From a97b461d034eb6c7011916985947a9e1c9ff0787 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Thu, 23 Mar 2023 11:10:37 +0100 Subject: [PATCH 01/66] chore: group extras and add instructions for pip installs Signed-off-by: anna-charlotte --- poetry.lock | 7 +++---- pyproject.toml | 9 ++++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2f89d1231dc..8c38a78ce56 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4016,13 +4016,12 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [extras] audio = ["pydub"] aws = ["smart-open"] -common = ["protobuf", "lz4"] -full = ["protobuf", "lz4", "pillow", "types-pillow", "av", "pydub", "trimesh"] +common = ["protobuf", "lz4", "pandas"] +full = ["protobuf", "lz4", "pandas", "pillow", "types-pillow", "av", "pydub", "trimesh"] hnswlib = ["hnswlib"] image = ["pillow", "types-pillow"] jac = ["jina-hubble-sdk"] mesh = ["trimesh"] -pandas = ["pandas"] torch = ["torch"] video = ["av"] web = ["fastapi"] @@ -4030,4 +4029,4 @@ web = ["fastapi"] [metadata] lock-version = "2.0" python-versions = ">=3.7,<4.0" -content-hash = "c5b13c9b48aa9edf9d494ce8ba91cfdd9f78d4220ae758ac8a74b69963fc7253" +content-hash = "3c0b9daa397c3f62d87970839a0d5cb84ddba0e96bd150dfa60c4c15d03fd1c3" diff --git a/pyproject.toml b/pyproject.toml index 9e30c3a804e..415c369ff5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,20 +28,19 @@ smart-open = {version = ">=6.3.0", extras = ["s3"], optional = true} jina-hubble-sdk = {version = ">=0.34.0", optional = true} [tool.poetry.extras] -common = ["protobuf", "lz4"] -torch = ["torch"] +common = ["protobuf", "lz4", "pandas"] image = ["pillow", "types-pillow"] video = ["av"] audio = ["pydub"] mesh = ["trimesh"] -web = ["fastapi"] hnswlib = ["hnswlib"] -pandas = ["pandas"] jac = ["jina-hubble-sdk"] aws = ["smart-open"] +torch = ["torch"] +web = ["fastapi"] # all -full = ["protobuf", "lz4", "pillow", "types-pillow", "av", "pydub", "trimesh"] +full = ["protobuf", "lz4", "pandas", "pillow", "types-pillow", "av", "pydub", "trimesh"] [tool.poetry.dev-dependencies] pytest = ">=7.0" From c346ef86e5f530c0a319b414b87a69387fac5630 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Thu, 23 Mar 2023 14:18:45 +0100 Subject: [PATCH 02/66] fix: throw runtime error with install instructions for hnswlib Signed-off-by: anna-charlotte --- docarray/index/backends/hnswlib.py | 10 +++++++-- docarray/utils/_internal/misc.py | 36 ++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index ab8aa6e56c1..725cba22669 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -4,6 +4,7 @@ from dataclasses import dataclass, field from pathlib import Path from typing import ( + TYPE_CHECKING, Any, Dict, Generic, @@ -17,7 +18,6 @@ cast, ) -import hnswlib import numpy as np from docarray import BaseDoc, DocArray @@ -29,10 +29,16 @@ _raise_not_supported, ) from docarray.proto import DocumentProto -from docarray.utils._internal.misc import is_np_int, is_tf_available, is_torch_available +from docarray.utils._internal.misc import import_library, is_np_int, is_tf_available, is_torch_available from docarray.utils.filter import filter_docs from docarray.utils.find import _FindResult +if TYPE_CHECKING: + import hnswlib +else: + hnswlib = import_library('hnswlib') + + TSchema = TypeVar('TSchema', bound=BaseDoc) T = TypeVar('T', bound='HnswDocumentIndex') diff --git a/docarray/utils/_internal/misc.py b/docarray/utils/_internal/misc.py index 99ed9b6696a..1ce225f7552 100644 --- a/docarray/utils/_internal/misc.py +++ b/docarray/utils/_internal/misc.py @@ -1,3 +1,4 @@ +import importlib from typing import Any import numpy as np @@ -17,11 +18,42 @@ else: tf_imported = True +import types +from typing import Optional + +INSTALL_INSTRUCTIONS = { + 'pillow': 'pip install docarray[image]', + 'pydub': 'pip install docarray[audio]', + 'av': 'pip install docarray[video]', + 'trimesh': 'pip install docarray[mesh]', + 'hnswlib': 'pip install docarray[hnswlib]', +} + + +def import_library(package: str) -> Optional[types.ModuleType]: + lib: Optional[types.ModuleType] + try: + lib = importlib.import_module(package) + except ModuleNotFoundError: + lib = None + + if lib is None: + raise RuntimeError( + 'Hnswlib is not installed. To install all the necessary libraries to use the hnsw backend, ' + f'please do: {INSTALL_INSTRUCTIONS[package]}.' + ) + else: + return lib + def is_torch_available(): return torch_imported +def is_tf_available(): + return tf_imported + + def is_np_int(item: Any) -> bool: dtype = getattr(item, 'dtype', None) ndim = getattr(item, 'ndim', None) @@ -33,10 +65,6 @@ def is_np_int(item: Any) -> bool: return False # this is unreachable, but mypy wants it -def is_tf_available(): - return tf_imported - - def is_notebook() -> bool: """ Check if we're running in a Jupyter notebook, using magic command From 9f4398516d9a93c64304e7f03beb2dd45ef5ab0f Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Fri, 24 Mar 2023 11:47:03 +0100 Subject: [PATCH 03/66] feat: add instructions for video imports Signed-off-by: anna-charlotte --- docarray/typing/bytes/video_bytes.py | 6 +++++- docarray/typing/tensor/video/video_tensor_mixin.py | 9 ++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/docarray/typing/bytes/video_bytes.py b/docarray/typing/bytes/video_bytes.py index fb45a36063f..079924b4a87 100644 --- a/docarray/typing/bytes/video_bytes.py +++ b/docarray/typing/bytes/video_bytes.py @@ -8,6 +8,7 @@ from docarray.typing import AudioNdArray, NdArray, VideoNdArray from docarray.typing.abstract_type import AbstractType from docarray.typing.proto_register import _register_proto +from docarray.utils.misc import import_library if TYPE_CHECKING: from pydantic.fields import BaseConfig, ModelField @@ -80,7 +81,10 @@ class MyDoc(BaseDoc): https://pyav.org/docs/stable/api/_globals.html?highlight=open#av.open :return: a VideoLoadResult instance with video, audio and keyframe indices """ - import av + if TYPE_CHECKING: + import av + else: + av = import_library('av') with av.open(BytesIO(self), **kwargs) as container: audio_frames = [] diff --git a/docarray/typing/tensor/video/video_tensor_mixin.py b/docarray/typing/tensor/video/video_tensor_mixin.py index 5dd11fb2da1..b204a22ca63 100644 --- a/docarray/typing/tensor/video/video_tensor_mixin.py +++ b/docarray/typing/tensor/video/video_tensor_mixin.py @@ -1,13 +1,13 @@ import abc import warnings from io import BytesIO -from typing import Optional, Type, TypeVar, Union +from typing import TYPE_CHECKING, Optional, Type, TypeVar, Union import numpy as np from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.audio.audio_tensor import AudioTensor -from docarray.utils._internal.misc import is_notebook +from docarray.utils._internal.misc import import_library, is_notebook T = TypeVar('T', bound='VideoTensorMixin') @@ -75,7 +75,10 @@ class MyDoc(BaseDoc): ) """ - import av + if TYPE_CHECKING: + import av + else: + av = import_library('av') np_tensor = self.get_comp_backend().to_numpy(array=self) video_tensor = np_tensor.astype('uint8') From d427c3f7df77b6bfc3e16f378b82e7ea98b3a2e6 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Fri, 24 Mar 2023 14:13:47 +0100 Subject: [PATCH 04/66] feat: add instructions for audio imports Signed-off-by: anna-charlotte --- docarray/typing/bytes/audio_bytes.py | 9 ++++++--- docarray/typing/tensor/audio/abstract_audio_tensor.py | 11 +++++++---- pyproject.toml | 4 ++++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/docarray/typing/bytes/audio_bytes.py b/docarray/typing/bytes/audio_bytes.py index 6a6c4e14bfd..01431093105 100644 --- a/docarray/typing/bytes/audio_bytes.py +++ b/docarray/typing/bytes/audio_bytes.py @@ -7,6 +7,7 @@ from docarray.typing.abstract_type import AbstractType from docarray.typing.proto_register import _register_proto +from docarray.utils.misc import import_library if TYPE_CHECKING: from pydantic.fields import BaseConfig, ModelField @@ -74,10 +75,12 @@ class MyAudio(Document): :return: np.ndarray representing the Audio as RGB values """ + if TYPE_CHECKING: + import pydub + else: + pydub = import_library('pydub') - from pydub import AudioSegment # type: ignore - - segment = AudioSegment.from_file(io.BytesIO(self)) + segment = pydub.AudioSegment.from_file(io.BytesIO(self)) # Convert to float32 using NumPy samples = np.array(segment.get_array_of_samples()) diff --git a/docarray/typing/tensor/audio/abstract_audio_tensor.py b/docarray/typing/tensor/audio/abstract_audio_tensor.py index e84b4ebc13e..40ecadacf0e 100644 --- a/docarray/typing/tensor/audio/abstract_audio_tensor.py +++ b/docarray/typing/tensor/audio/abstract_audio_tensor.py @@ -1,9 +1,9 @@ import warnings from abc import ABC -from typing import Any, BinaryIO, Dict, TypeVar, Union +from typing import TYPE_CHECKING, Any, BinaryIO, Dict, TypeVar, Union from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._internal.misc import is_notebook +from docarray.utils._internal.misc import import_library, is_notebook T = TypeVar('T', bound='AbstractAudioTensor') @@ -37,12 +37,15 @@ def save( :param sample_width: sample width in bytes :param pydub_args: dictionary of additional arguments for pydub.AudioSegment.export function """ - from pydub import AudioSegment # type: ignore + if TYPE_CHECKING: + import pydub + else: + pydub = import_library('pydub') comp_backend = self.get_comp_backend() channels = 2 if comp_backend.n_dim(array=self) > 1 else 1 # type: ignore - segment = AudioSegment( + segment = pydub.AudioSegment( self.to_bytes(), frame_rate=frame_rate, sample_width=sample_width, diff --git a/pyproject.toml b/pyproject.toml index 415c369ff5f..08030bf557d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,6 +113,10 @@ ignore_missing_imports = true module = "botocore" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "pydub" +ignore_missing_imports = true + [tool.black] skip-string-normalization = true # equivalent to black -S exclude = 'docarray/proto/pb*/*' From 86906880bca2372511588787ab9965cae60f08ee Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Fri, 24 Mar 2023 14:46:32 +0100 Subject: [PATCH 05/66] feat: add instructions for 3d imports Signed-off-by: anna-charlotte --- docarray/documents/mesh/vertices_and_faces.py | 9 +++++++-- docarray/documents/point_cloud/points_and_colors.py | 9 ++++++--- docarray/typing/url/url_3d/url_3d.py | 6 +++++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/docarray/documents/mesh/vertices_and_faces.py b/docarray/documents/mesh/vertices_and_faces.py index 364ad3187f6..294bb28e61f 100644 --- a/docarray/documents/mesh/vertices_and_faces.py +++ b/docarray/documents/mesh/vertices_and_faces.py @@ -1,7 +1,8 @@ -from typing import Any, Type, TypeVar, Union +from typing import TYPE_CHECKING, Any, Type, TypeVar, Union from docarray.base_doc import BaseDoc from docarray.typing.tensor.tensor import AnyTensor +from docarray.utils.misc import import_library T = TypeVar('T', bound='VerticesAndFaces') @@ -30,7 +31,11 @@ def display(self) -> None: Plot mesh consisting of vertices and faces. To use this you need to install trimesh[easy]: `pip install 'trimesh[easy]'`. """ - import trimesh + if TYPE_CHECKING: + import trimesh + else: + trimesh = import_library('trimesh') + from IPython.display import display if self.vertices is None or self.faces is None: diff --git a/docarray/documents/point_cloud/points_and_colors.py b/docarray/documents/point_cloud/points_and_colors.py index 82ce82f9ba1..49b70d28635 100644 --- a/docarray/documents/point_cloud/points_and_colors.py +++ b/docarray/documents/point_cloud/points_and_colors.py @@ -1,11 +1,11 @@ -from typing import Any, Optional, Type, TypeVar, Union +from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, Union import numpy as np from docarray.base_doc import BaseDoc from docarray.typing import AnyTensor from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._internal.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import import_library, is_tf_available, is_torch_available torch_available = is_torch_available() if torch_available: @@ -49,7 +49,10 @@ def display(self) -> None: Plot point cloud consisting of points in 3D space and optionally colors. To use this you need to install trimesh[easy]: `pip install 'trimesh[easy]'`. """ - import trimesh + if TYPE_CHECKING: + import trimesh + else: + trimesh = import_library('trimesh') from IPython.display import display colors = ( diff --git a/docarray/typing/url/url_3d/url_3d.py b/docarray/typing/url/url_3d/url_3d.py index 31228ccb124..a2210d8bd86 100644 --- a/docarray/typing/url/url_3d/url_3d.py +++ b/docarray/typing/url/url_3d/url_3d.py @@ -3,6 +3,7 @@ from docarray.typing.proto_register import _register_proto from docarray.typing.url.any_url import AnyUrl +from docarray.utils.misc import import_library if TYPE_CHECKING: import trimesh @@ -55,7 +56,10 @@ def _load_trimesh_instance( """ import urllib.parse - import trimesh + if TYPE_CHECKING: + import trimesh + else: + trimesh = import_library('trimesh') if not trimesh_args: trimesh_args = {} From c4b66732b437a25919eb6ebd31f4e788036de3e3 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Fri, 24 Mar 2023 15:30:26 +0100 Subject: [PATCH 06/66] feat: add instructions for image imports Signed-off-by: anna-charlotte --- docarray/typing/bytes/image_bytes.py | 6 +++++- .../tensor/image/abstract_image_tensor.py | 17 ++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docarray/typing/bytes/image_bytes.py b/docarray/typing/bytes/image_bytes.py index 2888f60fadd..8c1e2fa11ec 100644 --- a/docarray/typing/bytes/image_bytes.py +++ b/docarray/typing/bytes/image_bytes.py @@ -7,6 +7,7 @@ from docarray.typing.abstract_type import AbstractType from docarray.typing.proto_register import _register_proto +from docarray.utils.misc import import_library if TYPE_CHECKING: from pydantic.fields import BaseConfig, ModelField @@ -87,7 +88,10 @@ class MyDoc(BaseDoc): :return: np.ndarray representing the image as RGB values """ - from PIL import Image as PILImage + if TYPE_CHECKING: + from PIL import Image as PILImage + else: + PILImage = import_library('PIL.Image') raw_img = PILImage.open(BytesIO(self)) if width or height: diff --git a/docarray/typing/tensor/image/abstract_image_tensor.py b/docarray/typing/tensor/image/abstract_image_tensor.py index b02d1452f63..38768d8b8e5 100644 --- a/docarray/typing/tensor/image/abstract_image_tensor.py +++ b/docarray/typing/tensor/image/abstract_image_tensor.py @@ -1,9 +1,10 @@ import io import warnings from abc import ABC +from typing import TYPE_CHECKING from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._internal.misc import is_notebook +from docarray.utils._internal.misc import import_library, is_notebook class AbstractImageTensor(AbstractTensor, ABC): @@ -14,7 +15,10 @@ def to_bytes(self, format: str = 'PNG') -> bytes: :param format: the image format use to store the image, can be 'PNG' , 'JPG' ... :return: bytes """ - from PIL import Image + if TYPE_CHECKING: + from PIL import Image as PILImage + else: + PILImage = import_library('PIL.Image') if format == 'jpg': format = 'jpeg' # unify it to ISO standard @@ -22,7 +26,7 @@ def to_bytes(self, format: str = 'PNG') -> bytes: tensor = self.get_comp_backend().to_numpy(self) mode = 'RGB' if tensor.ndim == 3 else 'L' - pil_image = Image.fromarray(tensor, mode=mode) + pil_image = PILImage.fromarray(tensor, mode=mode) with io.BytesIO() as buffer: pil_image.save(buffer, format=format) @@ -35,10 +39,13 @@ def display(self) -> None: Display image data from tensor in notebook. """ if is_notebook(): - from PIL import Image + if TYPE_CHECKING: + import PIL + else: + PIL = import_library('PIL') np_array = self.get_comp_backend().to_numpy(self) - img = Image.fromarray(np_array) + img = PIL.Image.fromarray(np_array) from IPython.display import display From 2a0c350454587739634007a42c784334859df94c Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Fri, 24 Mar 2023 15:42:32 +0100 Subject: [PATCH 07/66] fix: import only audiosegment from pydub Signed-off-by: anna-charlotte --- docarray/typing/bytes/audio_bytes.py | 6 +++--- docarray/typing/tensor/audio/abstract_audio_tensor.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docarray/typing/bytes/audio_bytes.py b/docarray/typing/bytes/audio_bytes.py index 01431093105..ba939b0fd3c 100644 --- a/docarray/typing/bytes/audio_bytes.py +++ b/docarray/typing/bytes/audio_bytes.py @@ -76,11 +76,11 @@ class MyAudio(Document): :return: np.ndarray representing the Audio as RGB values """ if TYPE_CHECKING: - import pydub + from pydub import AudioSegment else: - pydub = import_library('pydub') + AudioSegment = import_library('pydub.AudioSegment') - segment = pydub.AudioSegment.from_file(io.BytesIO(self)) + segment = AudioSegment.from_file(io.BytesIO(self)) # Convert to float32 using NumPy samples = np.array(segment.get_array_of_samples()) diff --git a/docarray/typing/tensor/audio/abstract_audio_tensor.py b/docarray/typing/tensor/audio/abstract_audio_tensor.py index 40ecadacf0e..db2f4b9d301 100644 --- a/docarray/typing/tensor/audio/abstract_audio_tensor.py +++ b/docarray/typing/tensor/audio/abstract_audio_tensor.py @@ -38,14 +38,14 @@ def save( :param pydub_args: dictionary of additional arguments for pydub.AudioSegment.export function """ if TYPE_CHECKING: - import pydub + from pydub import AudioSegment else: - pydub = import_library('pydub') + AudioSegment = import_library('pydub.AudioSegment') comp_backend = self.get_comp_backend() channels = 2 if comp_backend.n_dim(array=self) > 1 else 1 # type: ignore - segment = pydub.AudioSegment( + segment = AudioSegment( self.to_bytes(), frame_rate=frame_rate, sample_width=sample_width, From 32428013a41570a47622674e134aca61f6cce8f6 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Fri, 24 Mar 2023 16:18:40 +0100 Subject: [PATCH 08/66] fix: generalize audio and image imports Signed-off-by: anna-charlotte --- docarray/typing/bytes/audio_bytes.py | 6 +++--- .../typing/tensor/audio/abstract_audio_tensor.py | 6 +++--- .../typing/tensor/image/abstract_image_tensor.py | 6 +++--- docarray/utils/_internal/misc.py | 14 +++++++------- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docarray/typing/bytes/audio_bytes.py b/docarray/typing/bytes/audio_bytes.py index ba939b0fd3c..01431093105 100644 --- a/docarray/typing/bytes/audio_bytes.py +++ b/docarray/typing/bytes/audio_bytes.py @@ -76,11 +76,11 @@ class MyAudio(Document): :return: np.ndarray representing the Audio as RGB values """ if TYPE_CHECKING: - from pydub import AudioSegment + import pydub else: - AudioSegment = import_library('pydub.AudioSegment') + pydub = import_library('pydub') - segment = AudioSegment.from_file(io.BytesIO(self)) + segment = pydub.AudioSegment.from_file(io.BytesIO(self)) # Convert to float32 using NumPy samples = np.array(segment.get_array_of_samples()) diff --git a/docarray/typing/tensor/audio/abstract_audio_tensor.py b/docarray/typing/tensor/audio/abstract_audio_tensor.py index db2f4b9d301..40ecadacf0e 100644 --- a/docarray/typing/tensor/audio/abstract_audio_tensor.py +++ b/docarray/typing/tensor/audio/abstract_audio_tensor.py @@ -38,14 +38,14 @@ def save( :param pydub_args: dictionary of additional arguments for pydub.AudioSegment.export function """ if TYPE_CHECKING: - from pydub import AudioSegment + import pydub else: - AudioSegment = import_library('pydub.AudioSegment') + pydub = import_library('pydub') comp_backend = self.get_comp_backend() channels = 2 if comp_backend.n_dim(array=self) > 1 else 1 # type: ignore - segment = AudioSegment( + segment = pydub.AudioSegment( self.to_bytes(), frame_rate=frame_rate, sample_width=sample_width, diff --git a/docarray/typing/tensor/image/abstract_image_tensor.py b/docarray/typing/tensor/image/abstract_image_tensor.py index 38768d8b8e5..02cec3e5a17 100644 --- a/docarray/typing/tensor/image/abstract_image_tensor.py +++ b/docarray/typing/tensor/image/abstract_image_tensor.py @@ -40,12 +40,12 @@ def display(self) -> None: """ if is_notebook(): if TYPE_CHECKING: - import PIL + from PIL import Image as PILImage else: - PIL = import_library('PIL') + PILImage = import_library('PIL.Image') np_array = self.get_comp_backend().to_numpy(self) - img = PIL.Image.fromarray(np_array) + img = PILImage.fromarray(np_array) from IPython.display import display diff --git a/docarray/utils/_internal/misc.py b/docarray/utils/_internal/misc.py index 1ce225f7552..e827413b818 100644 --- a/docarray/utils/_internal/misc.py +++ b/docarray/utils/_internal/misc.py @@ -22,11 +22,11 @@ from typing import Optional INSTALL_INSTRUCTIONS = { - 'pillow': 'pip install docarray[image]', - 'pydub': 'pip install docarray[audio]', - 'av': 'pip install docarray[video]', - 'trimesh': 'pip install docarray[mesh]', - 'hnswlib': 'pip install docarray[hnswlib]', + 'PIL.Image': 'pip install "docarray[image]"', + 'pydub': 'pip install "docarray[audio]"', + 'av': 'pip install "docarray[video]"', + 'trimesh': 'pip install "docarray[mesh]"', + 'hnswlib': 'pip install "docarray[hnswlib]"', } @@ -39,8 +39,8 @@ def import_library(package: str) -> Optional[types.ModuleType]: if lib is None: raise RuntimeError( - 'Hnswlib is not installed. To install all the necessary libraries to use the hnsw backend, ' - f'please do: {INSTALL_INSTRUCTIONS[package]}.' + f'{package} is not installed. To install all the necessary libraries to use the hnsw backend, ' + f'please do: `{INSTALL_INSTRUCTIONS[package]}`.' ) else: return lib From 391215d28fc4dd519452ecf03c10cd0b2b426ef3 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Fri, 24 Mar 2023 16:31:26 +0100 Subject: [PATCH 09/66] fix: add instructions for web imports Signed-off-by: anna-charlotte --- docarray/base_doc/doc_response.py | 21 +++++++++++++++++++-- docarray/utils/_internal/misc.py | 7 +++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/docarray/base_doc/doc_response.py b/docarray/base_doc/doc_response.py index cbc43bf4767..d45935474c9 100644 --- a/docarray/base_doc/doc_response.py +++ b/docarray/base_doc/doc_response.py @@ -1,14 +1,31 @@ from typing import Any +# from docarray.utils.misc import import_library + try: - from fastapi.responses import JSONResponse, Response + from fastapi.responses import JSONResponse except ImportError: class NoImportResponse: def __init__(self, *args, **kwargs): ImportError('fastapi is not installed') - Response = JSONResponse = NoImportResponse # type: ignore + JSONResponse = NoImportResponse # type: ignore + +# if TYPE_CHECKING: +# from fastapi.responses import JSONResponse, Response +# else: +# fastapi = import_library('fastapi', raise_error=False) +# if fastapi is None: +# class NoImportResponse: +# def __init__(self, *args, **kwargs): +# ImportError('fastapi is not installed') +# +# +# Response = JSONResponse = NoImportResponse # type: ignore +# else: +# from fastapi.responses import JSONResponse, Response +# class DocResponse(JSONResponse): diff --git a/docarray/utils/_internal/misc.py b/docarray/utils/_internal/misc.py index e827413b818..f5d1a6a35b2 100644 --- a/docarray/utils/_internal/misc.py +++ b/docarray/utils/_internal/misc.py @@ -27,17 +27,20 @@ 'av': 'pip install "docarray[video]"', 'trimesh': 'pip install "docarray[mesh]"', 'hnswlib': 'pip install "docarray[hnswlib]"', + 'fastapi': 'pip install "docarray[web]"', } -def import_library(package: str) -> Optional[types.ModuleType]: +def import_library( + package: str, raise_error: bool = True +) -> Optional[types.ModuleType]: lib: Optional[types.ModuleType] try: lib = importlib.import_module(package) except ModuleNotFoundError: lib = None - if lib is None: + if lib is None and raise_error: raise RuntimeError( f'{package} is not installed. To install all the necessary libraries to use the hnsw backend, ' f'please do: `{INSTALL_INSTRUCTIONS[package]}`.' From e4119b3ede86e9f074f41b301817c32660a614bf Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 08:19:44 +0200 Subject: [PATCH 10/66] fix: add instructions for web imports Signed-off-by: anna-charlotte --- docarray/base_doc/doc_response.py | 31 ++++++------------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/docarray/base_doc/doc_response.py b/docarray/base_doc/doc_response.py index d45935474c9..b8b646a10f6 100644 --- a/docarray/base_doc/doc_response.py +++ b/docarray/base_doc/doc_response.py @@ -1,31 +1,12 @@ -from typing import Any +from typing import TYPE_CHECKING, Any -# from docarray.utils.misc import import_library +from docarray.utils.misc import import_library -try: +if TYPE_CHECKING: + from fastapi.responses import JSONResponse +else: + fastapi = import_library('fastapi') from fastapi.responses import JSONResponse -except ImportError: - - class NoImportResponse: - def __init__(self, *args, **kwargs): - ImportError('fastapi is not installed') - - JSONResponse = NoImportResponse # type: ignore - -# if TYPE_CHECKING: -# from fastapi.responses import JSONResponse, Response -# else: -# fastapi = import_library('fastapi', raise_error=False) -# if fastapi is None: -# class NoImportResponse: -# def __init__(self, *args, **kwargs): -# ImportError('fastapi is not installed') -# -# -# Response = JSONResponse = NoImportResponse # type: ignore -# else: -# from fastapi.responses import JSONResponse, Response -# class DocResponse(JSONResponse): From 3eedfec261eb7d508b49cfc9c7e3ef336d92684f Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 08:40:31 +0200 Subject: [PATCH 11/66] fix: add instructions for protobuf imports Signed-off-by: anna-charlotte --- docarray/array/array/io.py | 6 +++++- docarray/proto/__init__.py | 11 ++++++++++- docarray/proto/pb/docarray_pb2.py | 27 ++++++++++++++++++++++----- docarray/proto/pb2/docarray_pb2.py | 29 ++++++++++++++++++++++------- docarray/utils/_internal/misc.py | 5 ++++- 5 files changed, 63 insertions(+), 15 deletions(-) diff --git a/docarray/array/array/io.py b/docarray/array/array/io.py index 1d8de498638..d6160cf9931 100644 --- a/docarray/array/array/io.py +++ b/docarray/array/array/io.py @@ -32,6 +32,7 @@ _dict_to_access_paths, ) from docarray.utils._internal.compress import _decompress_bytes, _get_compress_ctx +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: import pandas as pd @@ -500,7 +501,10 @@ def to_pandas(self) -> 'pd.DataFrame': :return: pandas.DataFrame """ - import pandas as pd + if TYPE_CHECKING: + import pandas as pd + else: + pd = import_library('pandas') fields = self.document_type._get_access_paths() df = pd.DataFrame(columns=fields) diff --git a/docarray/proto/__init__.py b/docarray/proto/__init__.py index 1f53c5e2ca8..391b0027277 100644 --- a/docarray/proto/__init__.py +++ b/docarray/proto/__init__.py @@ -1,4 +1,13 @@ -from google.protobuf import __version__ as __pb__version__ +from typing import TYPE_CHECKING + +from docarray.utils.misc import import_library + +if TYPE_CHECKING: + from google.protobuf import __version__ as __pb__version__ +else: + protobuf = import_library('google.protobuf') + __pb__version__ = protobuf.__version__ + if __pb__version__.startswith('4'): from docarray.proto.pb.docarray_pb2 import ( diff --git a/docarray/proto/pb/docarray_pb2.py b/docarray/proto/pb/docarray_pb2.py index 2f71d2fa67c..e856195cb51 100644 --- a/docarray/proto/pb/docarray_pb2.py +++ b/docarray/proto/pb/docarray_pb2.py @@ -2,16 +2,33 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: docarray.proto """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database +from typing import TYPE_CHECKING + +from docarray.utils.misc import import_library + +if TYPE_CHECKING: + from google.protobuf.internal import builder as _builder + from google.protobuf import descriptor as _descriptor + from google.protobuf import descriptor_pool as _descriptor_pool + from google.protobuf import symbol_database as _symbol_database +else: + protobuf = import_library('google.protobuf') + _builder = protobuf.internal.builder + _descriptor = protobuf.descriptor + _descriptor_pool = protobuf.descriptor_pool + _symbol_database = protobuf.symbol_database + # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() -from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 +if TYPE_CHECKING: + from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 +else: + protobuf = import_library('google.protobuf') + google_dot_protobuf_dot_struct__pb2 = protobuf.struct_pb2 + DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0e\x64ocarray.proto\x12\x08\x64ocarray\x1a\x1cgoogle/protobuf/struct.proto\"A\n\x11\x44\x65nseNdArrayProto\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\x0c\x12\r\n\x05shape\x18\x02 \x03(\r\x12\r\n\x05\x64type\x18\x03 \x01(\t\"g\n\x0cNdArrayProto\x12*\n\x05\x64\x65nse\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12+\n\nparameters\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\"Z\n\x0cKeyValuePair\x12#\n\x03key\x18\x01 \x01(\x0b\x32\x16.google.protobuf.Value\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value\";\n\x10GenericDictValue\x12\'\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x16.docarray.KeyValuePair\"\xc1\x03\n\tNodeProto\x12\x0e\n\x04text\x18\x01 \x01(\tH\x00\x12\x11\n\x07integer\x18\x02 \x01(\x05H\x00\x12\x0f\n\x05\x66loat\x18\x03 \x01(\x01H\x00\x12\x11\n\x07\x62oolean\x18\x04 \x01(\x08H\x00\x12\x0e\n\x04\x62lob\x18\x05 \x01(\x0cH\x00\x12)\n\x07ndarray\x18\x06 \x01(\x0b\x32\x16.docarray.NdArrayProtoH\x00\x12+\n\x08\x64ocument\x18\x07 \x01(\x0b\x32\x17.docarray.DocumentProtoH\x00\x12\x31\n\x0e\x64ocument_array\x18\x08 \x01(\x0b\x32\x17.docarray.DocArrayProtoH\x00\x12(\n\x04list\x18\t \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12\'\n\x03set\x18\n \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12)\n\x05tuple\x18\x0b \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12(\n\x04\x64ict\x18\x0c \x01(\x0b\x32\x18.docarray.DictOfAnyProtoH\x00\x12\x0e\n\x04type\x18\r \x01(\tH\x01\x42\t\n\x07\x63ontentB\x0f\n\rdocarray_type\"\x82\x01\n\rDocumentProto\x12/\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32!.docarray.DocumentProto.DataEntry\x1a@\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.docarray.NodeProto:\x02\x38\x01\"\x84\x01\n\x0e\x44ictOfAnyProto\x12\x30\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\".docarray.DictOfAnyProto.DataEntry\x1a@\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.docarray.NodeProto:\x02\x38\x01\"3\n\x0eListOfAnyProto\x12!\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x13.docarray.NodeProto\"6\n\rDocArrayProto\x12%\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x17.docarray.DocumentProto\"<\n\x13ListOfDocArrayProto\x12%\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x17.docarray.DocArrayProto\"\xed\x04\n\x14\x44ocArrayStackedProto\x12I\n\x0etensor_columns\x18\x01 \x03(\x0b\x32\x31.docarray.DocArrayStackedProto.TensorColumnsEntry\x12\x43\n\x0b\x64oc_columns\x18\x02 \x03(\x0b\x32..docarray.DocArrayStackedProto.DocColumnsEntry\x12\x41\n\nda_columns\x18\x03 \x03(\x0b\x32-.docarray.DocArrayStackedProto.DaColumnsEntry\x12\x43\n\x0b\x61ny_columns\x18\x04 \x03(\x0b\x32..docarray.DocArrayStackedProto.AnyColumnsEntry\x1aL\n\x12TensorColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.docarray.NdArrayProto:\x02\x38\x01\x1aQ\n\x0f\x44ocColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12-\n\x05value\x18\x02 \x01(\x0b\x32\x1e.docarray.DocArrayStackedProto:\x02\x38\x01\x1aO\n\x0e\x44\x61\x43olumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.docarray.ListOfDocArrayProto:\x02\x38\x01\x1aK\n\x0f\x41nyColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\'\n\x05value\x18\x02 \x01(\x0b\x32\x18.docarray.ListOfAnyProto:\x02\x38\x01\x62\x06proto3') diff --git a/docarray/proto/pb2/docarray_pb2.py b/docarray/proto/pb2/docarray_pb2.py index 2de3bbdf678..4fec4ef0e14 100644 --- a/docarray/proto/pb2/docarray_pb2.py +++ b/docarray/proto/pb2/docarray_pb2.py @@ -2,18 +2,33 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: docarray.proto """Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database +from typing import TYPE_CHECKING + +from docarray.utils.misc import import_library + +if TYPE_CHECKING: + from google.protobuf import descriptor as _descriptor + from google.protobuf import descriptor_pool as _descriptor_pool + from google.protobuf import message as _message + from google.protobuf import reflection as _reflection + from google.protobuf import symbol_database as _symbol_database +else: + protobuf = import_library('google.protobuf') + _descriptor = protobuf.descriptor + _descriptor_pool = protobuf.descriptor_pool + _message = protobuf.message + _reflection = protobuf.reflection + _symbol_database = protobuf.symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() - -from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 +if TYPE_CHECKING: + from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 +else: + protobuf = import_library('google.protobuf') + google_dot_protobuf_dot_struct__pb2 = protobuf.struct_pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( b'\n\x0e\x64ocarray.proto\x12\x08\x64ocarray\x1a\x1cgoogle/protobuf/struct.proto\"A\n\x11\x44\x65nseNdArrayProto\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\x0c\x12\r\n\x05shape\x18\x02 \x03(\r\x12\r\n\x05\x64type\x18\x03 \x01(\t\"g\n\x0cNdArrayProto\x12*\n\x05\x64\x65nse\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12+\n\nparameters\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\"Z\n\x0cKeyValuePair\x12#\n\x03key\x18\x01 \x01(\x0b\x32\x16.google.protobuf.Value\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value\";\n\x10GenericDictValue\x12\'\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x16.docarray.KeyValuePair\"\xc1\x03\n\tNodeProto\x12\x0e\n\x04text\x18\x01 \x01(\tH\x00\x12\x11\n\x07integer\x18\x02 \x01(\x05H\x00\x12\x0f\n\x05\x66loat\x18\x03 \x01(\x01H\x00\x12\x11\n\x07\x62oolean\x18\x04 \x01(\x08H\x00\x12\x0e\n\x04\x62lob\x18\x05 \x01(\x0cH\x00\x12)\n\x07ndarray\x18\x06 \x01(\x0b\x32\x16.docarray.NdArrayProtoH\x00\x12+\n\x08\x64ocument\x18\x07 \x01(\x0b\x32\x17.docarray.DocumentProtoH\x00\x12\x31\n\x0e\x64ocument_array\x18\x08 \x01(\x0b\x32\x17.docarray.DocArrayProtoH\x00\x12(\n\x04list\x18\t \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12\'\n\x03set\x18\n \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12)\n\x05tuple\x18\x0b \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12(\n\x04\x64ict\x18\x0c \x01(\x0b\x32\x18.docarray.DictOfAnyProtoH\x00\x12\x0e\n\x04type\x18\r \x01(\tH\x01\x42\t\n\x07\x63ontentB\x0f\n\rdocarray_type\"\x82\x01\n\rDocumentProto\x12/\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32!.docarray.DocumentProto.DataEntry\x1a@\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.docarray.NodeProto:\x02\x38\x01\"\x84\x01\n\x0e\x44ictOfAnyProto\x12\x30\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\".docarray.DictOfAnyProto.DataEntry\x1a@\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.docarray.NodeProto:\x02\x38\x01\"3\n\x0eListOfAnyProto\x12!\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x13.docarray.NodeProto\"6\n\rDocArrayProto\x12%\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x17.docarray.DocumentProto\"<\n\x13ListOfDocArrayProto\x12%\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x17.docarray.DocArrayProto\"\xed\x04\n\x14\x44ocArrayStackedProto\x12I\n\x0etensor_columns\x18\x01 \x03(\x0b\x32\x31.docarray.DocArrayStackedProto.TensorColumnsEntry\x12\x43\n\x0b\x64oc_columns\x18\x02 \x03(\x0b\x32..docarray.DocArrayStackedProto.DocColumnsEntry\x12\x41\n\nda_columns\x18\x03 \x03(\x0b\x32-.docarray.DocArrayStackedProto.DaColumnsEntry\x12\x43\n\x0b\x61ny_columns\x18\x04 \x03(\x0b\x32..docarray.DocArrayStackedProto.AnyColumnsEntry\x1aL\n\x12TensorColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.docarray.NdArrayProto:\x02\x38\x01\x1aQ\n\x0f\x44ocColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12-\n\x05value\x18\x02 \x01(\x0b\x32\x1e.docarray.DocArrayStackedProto:\x02\x38\x01\x1aO\n\x0e\x44\x61\x43olumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.docarray.ListOfDocArrayProto:\x02\x38\x01\x1aK\n\x0f\x41nyColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\'\n\x05value\x18\x02 \x01(\x0b\x32\x18.docarray.ListOfAnyProto:\x02\x38\x01\x62\x06proto3' diff --git a/docarray/utils/_internal/misc.py b/docarray/utils/_internal/misc.py index f5d1a6a35b2..7554cd6ad02 100644 --- a/docarray/utils/_internal/misc.py +++ b/docarray/utils/_internal/misc.py @@ -22,6 +22,9 @@ from typing import Optional INSTALL_INSTRUCTIONS = { + 'pandas': 'pip install "docarray[common]"', + 'google.protobuf': 'pip install "docarray[common]"', + 'lz4': 'pip install "docarray[common]"', 'PIL.Image': 'pip install "docarray[image]"', 'pydub': 'pip install "docarray[audio]"', 'av': 'pip install "docarray[video]"', @@ -37,7 +40,7 @@ def import_library( lib: Optional[types.ModuleType] try: lib = importlib.import_module(package) - except ModuleNotFoundError: + except (ModuleNotFoundError, ImportError): lib = None if lib is None and raise_error: From d027213408f1fe424a766f423820151857e86a91 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 08:53:51 +0200 Subject: [PATCH 12/66] fix: add instructions for lz4 imports Signed-off-by: anna-charlotte --- docarray/utils/_internal/compress.py | 27 ++++++++++++++++++++------- docarray/utils/_internal/misc.py | 2 +- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/docarray/utils/_internal/compress.py b/docarray/utils/_internal/compress.py index 0ff00f8eecd..7b98f64905d 100644 --- a/docarray/utils/_internal/compress.py +++ b/docarray/utils/_internal/compress.py @@ -1,11 +1,16 @@ -from typing import IO, Callable, Optional +from typing import IO, TYPE_CHECKING, Callable, Optional + +from docarray.utils._internal.misc import import_library def _compress_bytes(data: bytes, algorithm: Optional[str] = None) -> bytes: if algorithm == 'lz4': - import lz4.frame # type: ignore + if TYPE_CHECKING: + import lz4.frame as lz4_frame # type: ignore + else: + lz4_frame = import_library('lz4.frame') - data = lz4.frame.compress(data) + data = lz4_frame.compress(data) elif algorithm == 'bz2': import bz2 @@ -27,9 +32,13 @@ def _compress_bytes(data: bytes, algorithm: Optional[str] = None) -> bytes: def _decompress_bytes(data: bytes, algorithm: Optional[str] = None) -> bytes: if algorithm == 'lz4': - import lz4.frame # type: ignore - data = lz4.frame.decompress(data) + if TYPE_CHECKING: + import lz4.frame as lz4_frame # type: ignore + else: + lz4_frame = import_library('lz4.frame') + + data = lz4_frame.decompress(data) elif algorithm == 'bz2': import bz2 @@ -51,10 +60,14 @@ def _decompress_bytes(data: bytes, algorithm: Optional[str] = None) -> bytes: def _get_compress_ctx(algorithm: Optional[str] = None) -> Optional[Callable]: if algorithm == 'lz4': - import lz4.frame # type: ignore + + if TYPE_CHECKING: + import lz4.frame as lz4_frame # type: ignore + else: + lz4_frame = import_library('lz4.frame') def _fun(x: IO[bytes]): - return lz4.frame.LZ4FrameFile(x, 'wb') + return lz4_frame.LZ4FrameFile(x, 'wb') compress_ctx = _fun elif algorithm == 'gzip': diff --git a/docarray/utils/_internal/misc.py b/docarray/utils/_internal/misc.py index 7554cd6ad02..83d2b7863a0 100644 --- a/docarray/utils/_internal/misc.py +++ b/docarray/utils/_internal/misc.py @@ -24,7 +24,7 @@ INSTALL_INSTRUCTIONS = { 'pandas': 'pip install "docarray[common]"', 'google.protobuf': 'pip install "docarray[common]"', - 'lz4': 'pip install "docarray[common]"', + 'lz4.frame': 'pip install "docarray[common]"', 'PIL.Image': 'pip install "docarray[image]"', 'pydub': 'pip install "docarray[audio]"', 'av': 'pip install "docarray[video]"', From e5a5f0181021b6f0b938e95eef89760be7159655 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 09:13:34 +0200 Subject: [PATCH 13/66] fix: fastapi import Signed-off-by: anna-charlotte --- docarray/base_doc/doc_response.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docarray/base_doc/doc_response.py b/docarray/base_doc/doc_response.py index b8b646a10f6..c960cf6da4b 100644 --- a/docarray/base_doc/doc_response.py +++ b/docarray/base_doc/doc_response.py @@ -5,8 +5,16 @@ if TYPE_CHECKING: from fastapi.responses import JSONResponse else: - fastapi = import_library('fastapi') - from fastapi.responses import JSONResponse + fastapi = import_library('fastapi', raise_error=False) + if fastapi is None: + + class NoImportResponse: + def __init__(self, *args, **kwargs): + ImportError('fastapi is not installed') + + JSONResponse = NoImportResponse # type: ignore + else: + from fastapi.responses import JSONResponse class DocResponse(JSONResponse): From 9ffd5483711f6181cde65827869497e4fee46be8 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 09:28:54 +0200 Subject: [PATCH 14/66] fix: revert changes in protobuf import Signed-off-by: anna-charlotte --- docarray/proto/pb/docarray_pb2.py | 25 +++++-------------------- docarray/proto/pb2/docarray_pb2.py | 28 ++++++---------------------- 2 files changed, 11 insertions(+), 42 deletions(-) diff --git a/docarray/proto/pb/docarray_pb2.py b/docarray/proto/pb/docarray_pb2.py index e856195cb51..b3b0af6d3ac 100644 --- a/docarray/proto/pb/docarray_pb2.py +++ b/docarray/proto/pb/docarray_pb2.py @@ -2,33 +2,18 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: docarray.proto """Generated protocol buffer code.""" -from typing import TYPE_CHECKING -from docarray.utils.misc import import_library - -if TYPE_CHECKING: - from google.protobuf.internal import builder as _builder - from google.protobuf import descriptor as _descriptor - from google.protobuf import descriptor_pool as _descriptor_pool - from google.protobuf import symbol_database as _symbol_database -else: - protobuf = import_library('google.protobuf') - _builder = protobuf.internal.builder - _descriptor = protobuf.descriptor - _descriptor_pool = protobuf.descriptor_pool - _symbol_database = protobuf.symbol_database +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() -if TYPE_CHECKING: - from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 -else: - protobuf = import_library('google.protobuf') - google_dot_protobuf_dot_struct__pb2 = protobuf.struct_pb2 - +from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0e\x64ocarray.proto\x12\x08\x64ocarray\x1a\x1cgoogle/protobuf/struct.proto\"A\n\x11\x44\x65nseNdArrayProto\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\x0c\x12\r\n\x05shape\x18\x02 \x03(\r\x12\r\n\x05\x64type\x18\x03 \x01(\t\"g\n\x0cNdArrayProto\x12*\n\x05\x64\x65nse\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12+\n\nparameters\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\"Z\n\x0cKeyValuePair\x12#\n\x03key\x18\x01 \x01(\x0b\x32\x16.google.protobuf.Value\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value\";\n\x10GenericDictValue\x12\'\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x16.docarray.KeyValuePair\"\xc1\x03\n\tNodeProto\x12\x0e\n\x04text\x18\x01 \x01(\tH\x00\x12\x11\n\x07integer\x18\x02 \x01(\x05H\x00\x12\x0f\n\x05\x66loat\x18\x03 \x01(\x01H\x00\x12\x11\n\x07\x62oolean\x18\x04 \x01(\x08H\x00\x12\x0e\n\x04\x62lob\x18\x05 \x01(\x0cH\x00\x12)\n\x07ndarray\x18\x06 \x01(\x0b\x32\x16.docarray.NdArrayProtoH\x00\x12+\n\x08\x64ocument\x18\x07 \x01(\x0b\x32\x17.docarray.DocumentProtoH\x00\x12\x31\n\x0e\x64ocument_array\x18\x08 \x01(\x0b\x32\x17.docarray.DocArrayProtoH\x00\x12(\n\x04list\x18\t \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12\'\n\x03set\x18\n \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12)\n\x05tuple\x18\x0b \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12(\n\x04\x64ict\x18\x0c \x01(\x0b\x32\x18.docarray.DictOfAnyProtoH\x00\x12\x0e\n\x04type\x18\r \x01(\tH\x01\x42\t\n\x07\x63ontentB\x0f\n\rdocarray_type\"\x82\x01\n\rDocumentProto\x12/\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32!.docarray.DocumentProto.DataEntry\x1a@\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.docarray.NodeProto:\x02\x38\x01\"\x84\x01\n\x0e\x44ictOfAnyProto\x12\x30\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\".docarray.DictOfAnyProto.DataEntry\x1a@\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.docarray.NodeProto:\x02\x38\x01\"3\n\x0eListOfAnyProto\x12!\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x13.docarray.NodeProto\"6\n\rDocArrayProto\x12%\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x17.docarray.DocumentProto\"<\n\x13ListOfDocArrayProto\x12%\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x17.docarray.DocArrayProto\"\xed\x04\n\x14\x44ocArrayStackedProto\x12I\n\x0etensor_columns\x18\x01 \x03(\x0b\x32\x31.docarray.DocArrayStackedProto.TensorColumnsEntry\x12\x43\n\x0b\x64oc_columns\x18\x02 \x03(\x0b\x32..docarray.DocArrayStackedProto.DocColumnsEntry\x12\x41\n\nda_columns\x18\x03 \x03(\x0b\x32-.docarray.DocArrayStackedProto.DaColumnsEntry\x12\x43\n\x0b\x61ny_columns\x18\x04 \x03(\x0b\x32..docarray.DocArrayStackedProto.AnyColumnsEntry\x1aL\n\x12TensorColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.docarray.NdArrayProto:\x02\x38\x01\x1aQ\n\x0f\x44ocColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12-\n\x05value\x18\x02 \x01(\x0b\x32\x1e.docarray.DocArrayStackedProto:\x02\x38\x01\x1aO\n\x0e\x44\x61\x43olumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.docarray.ListOfDocArrayProto:\x02\x38\x01\x1aK\n\x0f\x41nyColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\'\n\x05value\x18\x02 \x01(\x0b\x32\x18.docarray.ListOfAnyProto:\x02\x38\x01\x62\x06proto3') diff --git a/docarray/proto/pb2/docarray_pb2.py b/docarray/proto/pb2/docarray_pb2.py index 4fec4ef0e14..f07dbd806b1 100644 --- a/docarray/proto/pb2/docarray_pb2.py +++ b/docarray/proto/pb2/docarray_pb2.py @@ -2,33 +2,17 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: docarray.proto """Generated protocol buffer code.""" -from typing import TYPE_CHECKING - -from docarray.utils.misc import import_library - -if TYPE_CHECKING: - from google.protobuf import descriptor as _descriptor - from google.protobuf import descriptor_pool as _descriptor_pool - from google.protobuf import message as _message - from google.protobuf import reflection as _reflection - from google.protobuf import symbol_database as _symbol_database -else: - protobuf = import_library('google.protobuf') - _descriptor = protobuf.descriptor - _descriptor_pool = protobuf.descriptor_pool - _message = protobuf.message - _reflection = protobuf.reflection - _symbol_database = protobuf.symbol_database +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() -if TYPE_CHECKING: - from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 -else: - protobuf = import_library('google.protobuf') - google_dot_protobuf_dot_struct__pb2 = protobuf.struct_pb2 +from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( b'\n\x0e\x64ocarray.proto\x12\x08\x64ocarray\x1a\x1cgoogle/protobuf/struct.proto\"A\n\x11\x44\x65nseNdArrayProto\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\x0c\x12\r\n\x05shape\x18\x02 \x03(\r\x12\r\n\x05\x64type\x18\x03 \x01(\t\"g\n\x0cNdArrayProto\x12*\n\x05\x64\x65nse\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12+\n\nparameters\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\"Z\n\x0cKeyValuePair\x12#\n\x03key\x18\x01 \x01(\x0b\x32\x16.google.protobuf.Value\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value\";\n\x10GenericDictValue\x12\'\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x16.docarray.KeyValuePair\"\xc1\x03\n\tNodeProto\x12\x0e\n\x04text\x18\x01 \x01(\tH\x00\x12\x11\n\x07integer\x18\x02 \x01(\x05H\x00\x12\x0f\n\x05\x66loat\x18\x03 \x01(\x01H\x00\x12\x11\n\x07\x62oolean\x18\x04 \x01(\x08H\x00\x12\x0e\n\x04\x62lob\x18\x05 \x01(\x0cH\x00\x12)\n\x07ndarray\x18\x06 \x01(\x0b\x32\x16.docarray.NdArrayProtoH\x00\x12+\n\x08\x64ocument\x18\x07 \x01(\x0b\x32\x17.docarray.DocumentProtoH\x00\x12\x31\n\x0e\x64ocument_array\x18\x08 \x01(\x0b\x32\x17.docarray.DocArrayProtoH\x00\x12(\n\x04list\x18\t \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12\'\n\x03set\x18\n \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12)\n\x05tuple\x18\x0b \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12(\n\x04\x64ict\x18\x0c \x01(\x0b\x32\x18.docarray.DictOfAnyProtoH\x00\x12\x0e\n\x04type\x18\r \x01(\tH\x01\x42\t\n\x07\x63ontentB\x0f\n\rdocarray_type\"\x82\x01\n\rDocumentProto\x12/\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32!.docarray.DocumentProto.DataEntry\x1a@\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.docarray.NodeProto:\x02\x38\x01\"\x84\x01\n\x0e\x44ictOfAnyProto\x12\x30\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\".docarray.DictOfAnyProto.DataEntry\x1a@\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.docarray.NodeProto:\x02\x38\x01\"3\n\x0eListOfAnyProto\x12!\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x13.docarray.NodeProto\"6\n\rDocArrayProto\x12%\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x17.docarray.DocumentProto\"<\n\x13ListOfDocArrayProto\x12%\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x17.docarray.DocArrayProto\"\xed\x04\n\x14\x44ocArrayStackedProto\x12I\n\x0etensor_columns\x18\x01 \x03(\x0b\x32\x31.docarray.DocArrayStackedProto.TensorColumnsEntry\x12\x43\n\x0b\x64oc_columns\x18\x02 \x03(\x0b\x32..docarray.DocArrayStackedProto.DocColumnsEntry\x12\x41\n\nda_columns\x18\x03 \x03(\x0b\x32-.docarray.DocArrayStackedProto.DaColumnsEntry\x12\x43\n\x0b\x61ny_columns\x18\x04 \x03(\x0b\x32..docarray.DocArrayStackedProto.AnyColumnsEntry\x1aL\n\x12TensorColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.docarray.NdArrayProto:\x02\x38\x01\x1aQ\n\x0f\x44ocColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12-\n\x05value\x18\x02 \x01(\x0b\x32\x1e.docarray.DocArrayStackedProto:\x02\x38\x01\x1aO\n\x0e\x44\x61\x43olumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.docarray.ListOfDocArrayProto:\x02\x38\x01\x1aK\n\x0f\x41nyColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\'\n\x05value\x18\x02 \x01(\x0b\x32\x18.docarray.ListOfAnyProto:\x02\x38\x01\x62\x06proto3' From 840b0bc73f2f79a2418b906ed00ccce843c39903 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 12:30:10 +0200 Subject: [PATCH 15/66] fix: add instructions for torch, without raising error Signed-off-by: anna-charlotte --- docarray/base_doc/mixins/io.py | 11 ++++++----- docarray/documents/audio.py | 11 ++++++----- docarray/documents/image.py | 12 +++++++----- docarray/documents/point_cloud/point_cloud_3d.py | 11 ++++++----- docarray/documents/point_cloud/points_and_colors.py | 9 +++++---- docarray/documents/video.py | 11 ++++++----- docarray/index/abstract.py | 8 +++++--- docarray/index/backends/hnswlib.py | 9 +++++++-- 8 files changed, 48 insertions(+), 34 deletions(-) diff --git a/docarray/base_doc/mixins/io.py b/docarray/base_doc/mixins/io.py index e0fde4b5d4f..1ff5c88c4c7 100644 --- a/docarray/base_doc/mixins/io.py +++ b/docarray/base_doc/mixins/io.py @@ -21,7 +21,7 @@ from docarray.typing import NdArray from docarray.typing.proto_register import _PROTO_TYPE_NAME_TO_CLASS from docarray.utils._internal.compress import _compress_bytes, _decompress_bytes -from docarray.utils._internal.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import import_library, is_tf_available tf_available = is_tf_available() if tf_available: @@ -29,10 +29,11 @@ from docarray.typing import TensorFlowTensor -torch_available = is_torch_available() -if torch_available: - import torch +if TYPE_CHECKING: + import torch +else: + torch = import_library('torch', raise_error=False) from docarray.typing import TorchTensor if TYPE_CHECKING: @@ -68,7 +69,7 @@ def _type_to_protobuf(value: Any) -> 'NodeProto': return nested_item base_node_wrap: BaseNode - if torch_available: + if torch is not None: if isinstance(value, torch.Tensor): base_node_wrap = TorchTensor._docarray_from_native(value) return base_node_wrap._to_node_protobuf() diff --git a/docarray/documents/audio.py b/docarray/documents/audio.py index e27abcb5408..8a68a5536ec 100644 --- a/docarray/documents/audio.py +++ b/docarray/documents/audio.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, Type, TypeVar, Union +from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, Union import numpy as np @@ -7,11 +7,12 @@ from docarray.typing.bytes.audio_bytes import AudioBytes from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.audio.audio_tensor import AudioTensor -from docarray.utils._internal.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import import_library, is_tf_available -torch_available = is_torch_available() -if torch_available: +if TYPE_CHECKING: import torch +else: + torch = import_library('torch', raise_error=False) tf_available = is_tf_available() if tf_available: @@ -110,7 +111,7 @@ def validate( if isinstance(value, str): value = cls(url=value) elif isinstance(value, (AbstractTensor, np.ndarray)) or ( - torch_available + torch is not None and isinstance(value, torch.Tensor) or (tf_available and isinstance(value, tf.Tensor)) ): diff --git a/docarray/documents/image.py b/docarray/documents/image.py index bf281f83d8f..a4b49984871 100644 --- a/docarray/documents/image.py +++ b/docarray/documents/image.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, Type, TypeVar, Union +from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, Union import numpy as np @@ -6,13 +6,15 @@ from docarray.typing import AnyEmbedding, ImageBytes, ImageUrl from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.image.image_tensor import ImageTensor -from docarray.utils._internal.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import import_library, is_tf_available T = TypeVar('T', bound='ImageDoc') -torch_available = is_torch_available() -if torch_available: +if TYPE_CHECKING: import torch +else: + torch = import_library('torch', raise_error=False) + tf_available = is_tf_available() if tf_available: @@ -99,7 +101,7 @@ def validate( value = cls(url=value) elif ( isinstance(value, (AbstractTensor, np.ndarray)) - or (torch_available and isinstance(value, torch.Tensor)) + or (torch is not None and isinstance(value, torch.Tensor)) or (tf_available and isinstance(value, tf.Tensor)) ): value = cls(tensor=value) diff --git a/docarray/documents/point_cloud/point_cloud_3d.py b/docarray/documents/point_cloud/point_cloud_3d.py index 25e0e6437d1..ffd834df0a6 100644 --- a/docarray/documents/point_cloud/point_cloud_3d.py +++ b/docarray/documents/point_cloud/point_cloud_3d.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, Type, TypeVar, Union +from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, Union import numpy as np @@ -6,11 +6,12 @@ from docarray.documents.point_cloud.points_and_colors import PointsAndColors from docarray.typing import AnyEmbedding, PointCloud3DUrl from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._internal.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import import_library, is_tf_available -torch_available = is_torch_available() -if torch_available: +if TYPE_CHECKING: import torch +else: + torch = import_library('torch', raise_error=False) tf_available = is_tf_available() if tf_available: @@ -124,7 +125,7 @@ def validate( if isinstance(value, str): value = cls(url=value) elif isinstance(value, (AbstractTensor, np.ndarray)) or ( - torch_available + torch is not None and isinstance(value, torch.Tensor) or (tf_available and isinstance(value, tf.Tensor)) ): diff --git a/docarray/documents/point_cloud/points_and_colors.py b/docarray/documents/point_cloud/points_and_colors.py index 49b70d28635..cd9abda6b79 100644 --- a/docarray/documents/point_cloud/points_and_colors.py +++ b/docarray/documents/point_cloud/points_and_colors.py @@ -5,11 +5,12 @@ from docarray.base_doc import BaseDoc from docarray.typing import AnyTensor from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._internal.misc import import_library, is_tf_available, is_torch_available +from docarray.utils._internal.misc import import_library, is_tf_available -torch_available = is_torch_available() -if torch_available: +if TYPE_CHECKING: import torch +else: + torch = import_library('torch', raise_error=False) tf_available = is_tf_available() if tf_available: @@ -36,7 +37,7 @@ def validate( value: Union[str, AbstractTensor, Any], ) -> T: if isinstance(value, (AbstractTensor, np.ndarray)) or ( - torch_available + torch is not None and isinstance(value, torch.Tensor) or (tf_available and isinstance(value, tf.Tensor)) ): diff --git a/docarray/documents/video.py b/docarray/documents/video.py index 679f10ea8ba..14ae1d0a034 100644 --- a/docarray/documents/video.py +++ b/docarray/documents/video.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, Type, TypeVar, Union +from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, Union import numpy as np @@ -8,11 +8,12 @@ from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.video.video_tensor import VideoTensor from docarray.typing.url.video_url import VideoUrl -from docarray.utils._internal.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import import_library, is_tf_available -torch_available = is_torch_available() -if torch_available: +if TYPE_CHECKING: import torch +else: + torch = import_library('torch', raise_error=False) tf_available = is_tf_available() @@ -112,7 +113,7 @@ def validate( if isinstance(value, str): value = cls(url=value) elif isinstance(value, (AbstractTensor, np.ndarray)) or ( - torch_available + torch is not None and isinstance(value, torch.Tensor) or (tf_available and isinstance(value, tf.Tensor)) ): diff --git a/docarray/index/abstract.py b/docarray/index/abstract.py index c587846c8c4..73a57ba7150 100644 --- a/docarray/index/abstract.py +++ b/docarray/index/abstract.py @@ -27,14 +27,16 @@ from docarray.array.abstract_array import AnyDocArray from docarray.typing import AnyTensor from docarray.utils._internal._typing import unwrap_optional_type -from docarray.utils._internal.misc import is_tf_available, torch_imported +from docarray.utils._internal.misc import import_library, is_tf_available from docarray.utils.find import FindResult, _FindResult if TYPE_CHECKING: from pydantic.fields import ModelField -if torch_imported: +if TYPE_CHECKING: import torch +else: + torch = import_library('torch', raise_error=False) if is_tf_available(): import tensorflow as tf # type: ignore @@ -818,7 +820,7 @@ def _to_numpy(self, val: Any, allow_passthrough=False) -> Any: return val.unwrap().numpy() if isinstance(val, (list, tuple)): return np.array(val) - if (torch_imported and isinstance(val, torch.Tensor)) or ( + if (torch is not None and isinstance(val, torch.Tensor)) or ( is_tf_available() and isinstance(val, tf.Tensor) ): return val.numpy() diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 725cba22669..265ab8d16d5 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -29,7 +29,7 @@ _raise_not_supported, ) from docarray.proto import DocumentProto -from docarray.utils._internal.misc import import_library, is_np_int, is_tf_available, is_torch_available +from docarray.utils._internal.misc import import_library, is_np_int, is_tf_available from docarray.utils.filter import filter_docs from docarray.utils.find import _FindResult @@ -43,11 +43,16 @@ T = TypeVar('T', bound='HnswDocumentIndex') HNSWLIB_PY_VEC_TYPES = [list, tuple, np.ndarray] -if is_torch_available(): + +if TYPE_CHECKING: import torch +else: + torch = import_library('torch', raise_error=False) +if torch is not None: HNSWLIB_PY_VEC_TYPES.append(torch.Tensor) + if is_tf_available(): import tensorflow as tf # type: ignore From e5da8c0a3999973dc670cb1d7b67aa50ee45622a Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 12:31:07 +0200 Subject: [PATCH 16/66] fix: add instructions for torch, with raising error Signed-off-by: anna-charlotte --- docarray/array/array/sequence_indexing_mixin.py | 9 +++++---- docarray/computation/torch_backend.py | 10 ++++++++-- docarray/typing/tensor/torch_tensor.py | 12 +++++++++--- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/docarray/array/array/sequence_indexing_mixin.py b/docarray/array/array/sequence_indexing_mixin.py index 01db74c0218..ac07359f3b0 100644 --- a/docarray/array/array/sequence_indexing_mixin.py +++ b/docarray/array/array/sequence_indexing_mixin.py @@ -1,5 +1,6 @@ import abc from typing import ( + TYPE_CHECKING, Any, Iterable, MutableSequence, @@ -14,7 +15,7 @@ import numpy as np -from docarray.utils._internal.misc import is_torch_available +from docarray.utils._internal.misc import import_library T_item = TypeVar('T_item') T = TypeVar('T', bound='IndexingSequenceMixin') @@ -85,11 +86,11 @@ def _normalize_index_item( return item.tolist() # torch index types - torch_available = is_torch_available() - if torch_available: + if TYPE_CHECKING: import torch else: - raise ValueError(f'Invalid index type {type(item)}') + torch = import_library('torch', raise_error=True) + allowed_torch_dtypes = [ torch.bool, torch.int64, diff --git a/docarray/computation/torch_backend.py b/docarray/computation/torch_backend.py index 9af155132cc..6526f1c8a89 100644 --- a/docarray/computation/torch_backend.py +++ b/docarray/computation/torch_backend.py @@ -1,7 +1,13 @@ -from typing import Any, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union import numpy as np -import torch + +from docarray.utils.misc import import_library + +if TYPE_CHECKING: + import torch +else: + torch = import_library('torch', raise_error=True) from docarray.computation.abstract_comp_backend import AbstractComputationalBackend diff --git a/docarray/typing/tensor/torch_tensor.py b/docarray/typing/tensor/torch_tensor.py index 7ca9cf70a89..a4c9e80a850 100644 --- a/docarray/typing/tensor/torch_tensor.py +++ b/docarray/typing/tensor/torch_tensor.py @@ -2,16 +2,22 @@ from typing import TYPE_CHECKING, Any, Dict, Generic, Type, TypeVar, Union, cast import numpy as np -import torch # type: ignore from docarray.typing.proto_register import _register_proto from docarray.typing.tensor.abstract_tensor import AbstractTensor +from docarray.utils.misc import import_library if TYPE_CHECKING: - from pydantic.fields import ModelField from pydantic import BaseConfig - from docarray.proto import NdArrayProto + from pydantic.fields import ModelField + from docarray.computation.torch_backend import TorchCompBackend + from docarray.proto import NdArrayProto + +if TYPE_CHECKING: + import torch +else: + torch = import_library('torch', raise_error=True) from docarray.base_doc.base_node import BaseNode From 41392bcf125cb42c6d025bed18471ec524a1e7fb Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 13:38:36 +0200 Subject: [PATCH 17/66] fix: add instructions for tensorflow Signed-off-by: anna-charlotte --- docarray/base_doc/mixins/io.py | 13 ++++++++----- docarray/computation/tensorflow_backend.py | 13 ++++++++++--- docarray/documents/audio.py | 9 +++++---- docarray/documents/image.py | 10 +++++----- docarray/documents/point_cloud/point_cloud_3d.py | 10 ++++++---- docarray/documents/point_cloud/points_and_colors.py | 9 +++++---- docarray/documents/video.py | 9 +++++---- docarray/index/abstract.py | 5 ++++- docarray/index/backends/hnswlib.py | 8 ++++++-- docarray/typing/tensor/tensorflow_tensor.py | 8 +++++++- docarray/utils/_internal/misc.py | 2 ++ 11 files changed, 63 insertions(+), 33 deletions(-) diff --git a/docarray/base_doc/mixins/io.py b/docarray/base_doc/mixins/io.py index 1ff5c88c4c7..1421aa54234 100644 --- a/docarray/base_doc/mixins/io.py +++ b/docarray/base_doc/mixins/io.py @@ -21,17 +21,20 @@ from docarray.typing import NdArray from docarray.typing.proto_register import _PROTO_TYPE_NAME_TO_CLASS from docarray.utils._internal.compress import _compress_bytes, _decompress_bytes -from docarray.utils._internal.misc import import_library, is_tf_available +from docarray.utils._internal.misc import import_library -tf_available = is_tf_available() -if tf_available: +if TYPE_CHECKING: import tensorflow as tf # type: ignore from docarray.typing import TensorFlowTensor - +else: + tf = import_library('tensorflow', raise_error=True) + from docarray.typing import TensorFlowTensor if TYPE_CHECKING: import torch + + from docarray.typing import TorchTensor else: torch = import_library('torch', raise_error=False) from docarray.typing import TorchTensor @@ -74,7 +77,7 @@ def _type_to_protobuf(value: Any) -> 'NodeProto': base_node_wrap = TorchTensor._docarray_from_native(value) return base_node_wrap._to_node_protobuf() - if tf_available: + if tf is not None: if isinstance(value, tf.Tensor): base_node_wrap = TensorFlowTensor._docarray_from_native(value) return base_node_wrap._to_node_protobuf() diff --git a/docarray/computation/tensorflow_backend.py b/docarray/computation/tensorflow_backend.py index f4fc9edaa51..d229e7f6d28 100644 --- a/docarray/computation/tensorflow_backend.py +++ b/docarray/computation/tensorflow_backend.py @@ -1,9 +1,16 @@ import typing -from typing import Callable, List, Optional, Tuple +from typing import TYPE_CHECKING, Callable, List, Optional, Tuple import numpy as np -import tensorflow as tf # type: ignore -import tensorflow._api.v2.experimental.numpy as tnp # type: ignore + +from docarray.utils.misc import import_library + +if TYPE_CHECKING: + import tensorflow as tf # type: ignore + import tensorflow._api.v2.experimental.numpy as tnp # type: ignore +else: + tf = import_library('tensorflow', raise_error=True) + tnp = tf._api.v2.experimental.numpy from docarray.computation import AbstractComputationalBackend from docarray.computation.abstract_numpy_based_backend import AbstractNumpyBasedBackend diff --git a/docarray/documents/audio.py b/docarray/documents/audio.py index 8a68a5536ec..ea67402c5cd 100644 --- a/docarray/documents/audio.py +++ b/docarray/documents/audio.py @@ -7,16 +7,17 @@ from docarray.typing.bytes.audio_bytes import AudioBytes from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.audio.audio_tensor import AudioTensor -from docarray.utils._internal.misc import import_library, is_tf_available +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: import torch else: torch = import_library('torch', raise_error=False) -tf_available = is_tf_available() -if tf_available: +if TYPE_CHECKING: import tensorflow as tf # type: ignore +else: + tf = import_library('tensorflow', raise_error=False) T = TypeVar('T', bound='AudioDoc') @@ -113,7 +114,7 @@ def validate( elif isinstance(value, (AbstractTensor, np.ndarray)) or ( torch is not None and isinstance(value, torch.Tensor) - or (tf_available and isinstance(value, tf.Tensor)) + or (tf is not None and isinstance(value, tf.Tensor)) ): value = cls(tensor=value) diff --git a/docarray/documents/image.py b/docarray/documents/image.py index a4b49984871..369855f75a9 100644 --- a/docarray/documents/image.py +++ b/docarray/documents/image.py @@ -6,7 +6,7 @@ from docarray.typing import AnyEmbedding, ImageBytes, ImageUrl from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.image.image_tensor import ImageTensor -from docarray.utils._internal.misc import import_library, is_tf_available +from docarray.utils._internal.misc import import_library T = TypeVar('T', bound='ImageDoc') @@ -15,10 +15,10 @@ else: torch = import_library('torch', raise_error=False) - -tf_available = is_tf_available() -if tf_available: +if TYPE_CHECKING: import tensorflow as tf # type: ignore +else: + tf = import_library('tensorflow', raise_error=False) class ImageDoc(BaseDoc): @@ -102,7 +102,7 @@ def validate( elif ( isinstance(value, (AbstractTensor, np.ndarray)) or (torch is not None and isinstance(value, torch.Tensor)) - or (tf_available and isinstance(value, tf.Tensor)) + or (tf is not None and isinstance(value, tf.Tensor)) ): value = cls(tensor=value) elif isinstance(value, bytes): diff --git a/docarray/documents/point_cloud/point_cloud_3d.py b/docarray/documents/point_cloud/point_cloud_3d.py index ffd834df0a6..ca0ff2f6da8 100644 --- a/docarray/documents/point_cloud/point_cloud_3d.py +++ b/docarray/documents/point_cloud/point_cloud_3d.py @@ -6,16 +6,18 @@ from docarray.documents.point_cloud.points_and_colors import PointsAndColors from docarray.typing import AnyEmbedding, PointCloud3DUrl from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._internal.misc import import_library, is_tf_available +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: import torch else: torch = import_library('torch', raise_error=False) -tf_available = is_tf_available() -if tf_available: + +if TYPE_CHECKING: import tensorflow as tf # type: ignore +else: + tf = import_library('tensorflow', raise_error=False) T = TypeVar('T', bound='PointCloud3D') @@ -127,7 +129,7 @@ def validate( elif isinstance(value, (AbstractTensor, np.ndarray)) or ( torch is not None and isinstance(value, torch.Tensor) - or (tf_available and isinstance(value, tf.Tensor)) + or (tf is not None and isinstance(value, tf.Tensor)) ): value = cls(tensors=PointsAndColors(points=value)) diff --git a/docarray/documents/point_cloud/points_and_colors.py b/docarray/documents/point_cloud/points_and_colors.py index cd9abda6b79..4fee548f5a3 100644 --- a/docarray/documents/point_cloud/points_and_colors.py +++ b/docarray/documents/point_cloud/points_and_colors.py @@ -5,16 +5,17 @@ from docarray.base_doc import BaseDoc from docarray.typing import AnyTensor from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._internal.misc import import_library, is_tf_available +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: import torch else: torch = import_library('torch', raise_error=False) -tf_available = is_tf_available() -if tf_available: +if TYPE_CHECKING: import tensorflow as tf # type: ignore +else: + tf = import_library('tensorflow', raise_error=False) T = TypeVar('T', bound='PointsAndColors') @@ -39,7 +40,7 @@ def validate( if isinstance(value, (AbstractTensor, np.ndarray)) or ( torch is not None and isinstance(value, torch.Tensor) - or (tf_available and isinstance(value, tf.Tensor)) + or (tf is not None and isinstance(value, tf.Tensor)) ): value = cls(points=value) diff --git a/docarray/documents/video.py b/docarray/documents/video.py index 14ae1d0a034..ec536fd970d 100644 --- a/docarray/documents/video.py +++ b/docarray/documents/video.py @@ -8,7 +8,7 @@ from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.typing.tensor.video.video_tensor import VideoTensor from docarray.typing.url.video_url import VideoUrl -from docarray.utils._internal.misc import import_library, is_tf_available +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: import torch @@ -16,9 +16,10 @@ torch = import_library('torch', raise_error=False) -tf_available = is_tf_available() -if tf_available: +if TYPE_CHECKING: import tensorflow as tf # type: ignore +else: + tf = import_library('tensorflow', raise_error=False) T = TypeVar('T', bound='VideoDoc') @@ -115,7 +116,7 @@ def validate( elif isinstance(value, (AbstractTensor, np.ndarray)) or ( torch is not None and isinstance(value, torch.Tensor) - or (tf_available and isinstance(value, tf.Tensor)) + or (tf is not None and isinstance(value, tf.Tensor)) ): value = cls(tensor=value) diff --git a/docarray/index/abstract.py b/docarray/index/abstract.py index 73a57ba7150..c889c0ecb92 100644 --- a/docarray/index/abstract.py +++ b/docarray/index/abstract.py @@ -38,10 +38,13 @@ else: torch = import_library('torch', raise_error=False) -if is_tf_available(): +if TYPE_CHECKING: import tensorflow as tf # type: ignore from docarray.typing import TensorFlowTensor +else: + tf = import_library('tensorflow', raise_error=False) + from docarray.typing import TensorFlowTensor TSchema = TypeVar('TSchema', bound=BaseDoc) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 265ab8d16d5..7a0433f9226 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -29,7 +29,7 @@ _raise_not_supported, ) from docarray.proto import DocumentProto -from docarray.utils._internal.misc import import_library, is_np_int, is_tf_available +from docarray.utils._internal.misc import import_library, is_np_int from docarray.utils.filter import filter_docs from docarray.utils.find import _FindResult @@ -53,11 +53,15 @@ HNSWLIB_PY_VEC_TYPES.append(torch.Tensor) -if is_tf_available(): +if TYPE_CHECKING: import tensorflow as tf # type: ignore from docarray.typing import TensorFlowTensor +else: + tf = import_library('tensorflow', raise_error=False) + from docarray.typing import TensorFlowTensor +if tf is not None: HNSWLIB_PY_VEC_TYPES.append(tf.Tensor) HNSWLIB_PY_VEC_TYPES.append(TensorFlowTensor) diff --git a/docarray/typing/tensor/tensorflow_tensor.py b/docarray/typing/tensor/tensorflow_tensor.py index 4f28ab78a47..e19cd05941c 100644 --- a/docarray/typing/tensor/tensorflow_tensor.py +++ b/docarray/typing/tensor/tensorflow_tensor.py @@ -1,7 +1,13 @@ from typing import TYPE_CHECKING, Any, Dict, Generic, Type, TypeVar, Union, cast import numpy as np -import tensorflow as tf # type: ignore + +from docarray.utils.misc import import_library + +if TYPE_CHECKING: + import tensorflow as tf # type: ignore +else: + tf = import_library('tensorflow', raise_error=True) from docarray.typing.proto_register import _register_proto from docarray.typing.tensor.abstract_tensor import AbstractTensor diff --git a/docarray/utils/_internal/misc.py b/docarray/utils/_internal/misc.py index 83d2b7863a0..f7af860082b 100644 --- a/docarray/utils/_internal/misc.py +++ b/docarray/utils/_internal/misc.py @@ -31,6 +31,8 @@ 'trimesh': 'pip install "docarray[mesh]"', 'hnswlib': 'pip install "docarray[hnswlib]"', 'fastapi': 'pip install "docarray[web]"', + 'torch': 'pip install "docarray[torch]"', + 'tensorflow': 'pip install pip install protobuf==3.19.0 tensorflow', } From 581206a719f323b6595003adc9ef8b26290d8c90 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 14:04:23 +0200 Subject: [PATCH 18/66] fix: base doc io imports Signed-off-by: anna-charlotte --- docarray/base_doc/mixins/io.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docarray/base_doc/mixins/io.py b/docarray/base_doc/mixins/io.py index 1421aa54234..66bd6251400 100644 --- a/docarray/base_doc/mixins/io.py +++ b/docarray/base_doc/mixins/io.py @@ -28,8 +28,9 @@ from docarray.typing import TensorFlowTensor else: - tf = import_library('tensorflow', raise_error=True) - from docarray.typing import TensorFlowTensor + tf = import_library('tensorflow', raise_error=False) + if tf is not None: + from docarray.typing import TensorFlowTensor if TYPE_CHECKING: import torch @@ -37,7 +38,8 @@ from docarray.typing import TorchTensor else: torch = import_library('torch', raise_error=False) - from docarray.typing import TorchTensor + if torch is not None: + from docarray.typing import TorchTensor if TYPE_CHECKING: from pydantic.fields import ModelField From 59452b8d4c35e2d962baa02483dfe4f981b90435 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 14:27:28 +0200 Subject: [PATCH 19/66] fix: tf in doc index abstract Signed-off-by: anna-charlotte --- docarray/index/abstract.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docarray/index/abstract.py b/docarray/index/abstract.py index c889c0ecb92..ab6ceb96db3 100644 --- a/docarray/index/abstract.py +++ b/docarray/index/abstract.py @@ -44,7 +44,8 @@ from docarray.typing import TensorFlowTensor else: tf = import_library('tensorflow', raise_error=False) - from docarray.typing import TensorFlowTensor + if tf is not None: + from docarray.typing import TensorFlowTensor TSchema = TypeVar('TSchema', bound=BaseDoc) From 49e675966ed1841c04ebed4e304965110eafa138 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 14:28:49 +0200 Subject: [PATCH 20/66] fix: tf in doc index abstract Signed-off-by: anna-charlotte --- docarray/index/abstract.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docarray/index/abstract.py b/docarray/index/abstract.py index ab6ceb96db3..3f26e81d238 100644 --- a/docarray/index/abstract.py +++ b/docarray/index/abstract.py @@ -27,7 +27,7 @@ from docarray.array.abstract_array import AnyDocArray from docarray.typing import AnyTensor from docarray.utils._internal._typing import unwrap_optional_type -from docarray.utils._internal.misc import import_library, is_tf_available +from docarray.utils._internal.misc import import_library from docarray.utils.find import FindResult, _FindResult if TYPE_CHECKING: @@ -820,12 +820,12 @@ def _to_numpy(self, val: Any, allow_passthrough=False) -> Any: """ if isinstance(val, np.ndarray): return val - if is_tf_available() and isinstance(val, TensorFlowTensor): + if tf is not None and isinstance(val, TensorFlowTensor): return val.unwrap().numpy() if isinstance(val, (list, tuple)): return np.array(val) if (torch is not None and isinstance(val, torch.Tensor)) or ( - is_tf_available() and isinstance(val, tf.Tensor) + tf is not None and isinstance(val, tf.Tensor) ): return val.numpy() if allow_passthrough: From 39793400a8525566b1887d8395d0ac49c6556287 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 15:19:38 +0200 Subject: [PATCH 21/66] fix: clean up imports Signed-off-by: anna-charlotte --- docarray/base_doc/mixins/io.py | 15 +++------ docarray/computation/tensorflow_backend.py | 7 ++--- docarray/computation/torch_backend.py | 3 +- docarray/documents/audio.py | 5 +-- docarray/documents/image.py | 9 ++---- .../documents/point_cloud/point_cloud_3d.py | 7 ++--- .../point_cloud/points_and_colors.py | 6 ++-- docarray/documents/video.py | 8 ++--- docarray/index/abstract.py | 11 ++----- docarray/index/backends/hnswlib.py | 31 +++++++------------ docarray/typing/tensor/tensorflow_tensor.py | 17 +++++----- docarray/typing/tensor/torch_tensor.py | 6 ++-- docarray/utils/_internal/misc.py | 7 +++-- 13 files changed, 47 insertions(+), 85 deletions(-) diff --git a/docarray/base_doc/mixins/io.py b/docarray/base_doc/mixins/io.py index 66bd6251400..9654ae03d41 100644 --- a/docarray/base_doc/mixins/io.py +++ b/docarray/base_doc/mixins/io.py @@ -25,27 +25,20 @@ if TYPE_CHECKING: import tensorflow as tf # type: ignore + import torch + from pydantic.fields import ModelField - from docarray.typing import TensorFlowTensor + from docarray.proto import DocumentProto, NodeProto + from docarray.typing import TensorFlowTensor, TorchTensor else: tf = import_library('tensorflow', raise_error=False) if tf is not None: from docarray.typing import TensorFlowTensor -if TYPE_CHECKING: - import torch - - from docarray.typing import TorchTensor -else: torch = import_library('torch', raise_error=False) if torch is not None: from docarray.typing import TorchTensor -if TYPE_CHECKING: - from pydantic.fields import ModelField - - from docarray.proto import DocumentProto, NodeProto - T = TypeVar('T', bound='IOMixin') diff --git a/docarray/computation/tensorflow_backend.py b/docarray/computation/tensorflow_backend.py index d229e7f6d28..41341e3066b 100644 --- a/docarray/computation/tensorflow_backend.py +++ b/docarray/computation/tensorflow_backend.py @@ -3,6 +3,9 @@ import numpy as np +from docarray.computation import AbstractComputationalBackend +from docarray.computation.abstract_numpy_based_backend import AbstractNumpyBasedBackend +from docarray.typing import TensorFlowTensor from docarray.utils.misc import import_library if TYPE_CHECKING: @@ -12,10 +15,6 @@ tf = import_library('tensorflow', raise_error=True) tnp = tf._api.v2.experimental.numpy -from docarray.computation import AbstractComputationalBackend -from docarray.computation.abstract_numpy_based_backend import AbstractNumpyBasedBackend -from docarray.typing import TensorFlowTensor - def _unsqueeze_if_single_axis(*matrices: tf.Tensor) -> List[tf.Tensor]: """ diff --git a/docarray/computation/torch_backend.py b/docarray/computation/torch_backend.py index 6526f1c8a89..0871fc2e689 100644 --- a/docarray/computation/torch_backend.py +++ b/docarray/computation/torch_backend.py @@ -2,6 +2,7 @@ import numpy as np +from docarray.computation.abstract_comp_backend import AbstractComputationalBackend from docarray.utils.misc import import_library if TYPE_CHECKING: @@ -9,8 +10,6 @@ else: torch = import_library('torch', raise_error=True) -from docarray.computation.abstract_comp_backend import AbstractComputationalBackend - def _unsqueeze_if_single_axis(*matrices: torch.Tensor) -> List[torch.Tensor]: """Unsqueezes tensors that only have one axis, at dim 0. diff --git a/docarray/documents/audio.py b/docarray/documents/audio.py index ea67402c5cd..3103ab656c1 100644 --- a/docarray/documents/audio.py +++ b/docarray/documents/audio.py @@ -10,13 +10,10 @@ from docarray.utils._internal.misc import import_library if TYPE_CHECKING: + import tensorflow as tf # type: ignore import torch else: torch = import_library('torch', raise_error=False) - -if TYPE_CHECKING: - import tensorflow as tf # type: ignore -else: tf = import_library('tensorflow', raise_error=False) diff --git a/docarray/documents/image.py b/docarray/documents/image.py index 369855f75a9..47456c8cd29 100644 --- a/docarray/documents/image.py +++ b/docarray/documents/image.py @@ -8,17 +8,14 @@ from docarray.typing.tensor.image.image_tensor import ImageTensor from docarray.utils._internal.misc import import_library -T = TypeVar('T', bound='ImageDoc') - if TYPE_CHECKING: + import tensorflow as tf # type: ignore import torch else: + tf = import_library('tensorflow', raise_error=False) torch = import_library('torch', raise_error=False) -if TYPE_CHECKING: - import tensorflow as tf # type: ignore -else: - tf = import_library('tensorflow', raise_error=False) +T = TypeVar('T', bound='ImageDoc') class ImageDoc(BaseDoc): diff --git a/docarray/documents/point_cloud/point_cloud_3d.py b/docarray/documents/point_cloud/point_cloud_3d.py index ca0ff2f6da8..36e56a4c571 100644 --- a/docarray/documents/point_cloud/point_cloud_3d.py +++ b/docarray/documents/point_cloud/point_cloud_3d.py @@ -9,16 +9,13 @@ from docarray.utils._internal.misc import import_library if TYPE_CHECKING: + import tensorflow as tf # type: ignore import torch else: + tf = import_library('tensorflow', raise_error=False) torch = import_library('torch', raise_error=False) -if TYPE_CHECKING: - import tensorflow as tf # type: ignore -else: - tf = import_library('tensorflow', raise_error=False) - T = TypeVar('T', bound='PointCloud3D') diff --git a/docarray/documents/point_cloud/points_and_colors.py b/docarray/documents/point_cloud/points_and_colors.py index 4fee548f5a3..288c8caba3e 100644 --- a/docarray/documents/point_cloud/points_and_colors.py +++ b/docarray/documents/point_cloud/points_and_colors.py @@ -8,15 +8,13 @@ from docarray.utils._internal.misc import import_library if TYPE_CHECKING: + import tensorflow as tf # type: ignore import torch else: torch = import_library('torch', raise_error=False) - -if TYPE_CHECKING: - import tensorflow as tf # type: ignore -else: tf = import_library('tensorflow', raise_error=False) + T = TypeVar('T', bound='PointsAndColors') diff --git a/docarray/documents/video.py b/docarray/documents/video.py index ec536fd970d..1035beac277 100644 --- a/docarray/documents/video.py +++ b/docarray/documents/video.py @@ -10,16 +10,12 @@ from docarray.typing.url.video_url import VideoUrl from docarray.utils._internal.misc import import_library -if TYPE_CHECKING: - import torch -else: - torch = import_library('torch', raise_error=False) - - if TYPE_CHECKING: import tensorflow as tf # type: ignore + import torch else: tf = import_library('tensorflow', raise_error=False) + torch = import_library('torch', raise_error=False) T = TypeVar('T', bound='VideoDoc') diff --git a/docarray/index/abstract.py b/docarray/index/abstract.py index 3f26e81d238..1f4deceaa8b 100644 --- a/docarray/index/abstract.py +++ b/docarray/index/abstract.py @@ -30,22 +30,17 @@ from docarray.utils._internal.misc import import_library from docarray.utils.find import FindResult, _FindResult -if TYPE_CHECKING: - from pydantic.fields import ModelField - -if TYPE_CHECKING: - import torch -else: - torch = import_library('torch', raise_error=False) - if TYPE_CHECKING: import tensorflow as tf # type: ignore + import torch + from pydantic.fields import ModelField from docarray.typing import TensorFlowTensor else: tf = import_library('tensorflow', raise_error=False) if tf is not None: from docarray.typing import TensorFlowTensor + torch = import_library('torch', raise_error=False) TSchema = TypeVar('TSchema', bound=BaseDoc) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 7a0433f9226..4e5fa1db682 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -35,37 +35,30 @@ if TYPE_CHECKING: import hnswlib -else: - hnswlib = import_library('hnswlib') - - -TSchema = TypeVar('TSchema', bound=BaseDoc) -T = TypeVar('T', bound='HnswDocumentIndex') - -HNSWLIB_PY_VEC_TYPES = [list, tuple, np.ndarray] - -if TYPE_CHECKING: - import torch -else: - torch = import_library('torch', raise_error=False) - -if torch is not None: - HNSWLIB_PY_VEC_TYPES.append(torch.Tensor) - - -if TYPE_CHECKING: import tensorflow as tf # type: ignore + import torch from docarray.typing import TensorFlowTensor else: + hnswlib = import_library('hnswlib', raise_error=True) + torch = import_library('torch', raise_error=False) tf = import_library('tensorflow', raise_error=False) from docarray.typing import TensorFlowTensor +HNSWLIB_PY_VEC_TYPES = [list, tuple, np.ndarray] + +if torch is not None: + HNSWLIB_PY_VEC_TYPES.append(torch.Tensor) + if tf is not None: HNSWLIB_PY_VEC_TYPES.append(tf.Tensor) HNSWLIB_PY_VEC_TYPES.append(TensorFlowTensor) +TSchema = TypeVar('TSchema', bound=BaseDoc) +T = TypeVar('T', bound='HnswDocumentIndex') + + def _collect_query_args(method_name: str): # TODO: use partialmethod instead def inner(self, *args, **kwargs): if args: diff --git a/docarray/typing/tensor/tensorflow_tensor.py b/docarray/typing/tensor/tensorflow_tensor.py index e19cd05941c..6d60ea3d629 100644 --- a/docarray/typing/tensor/tensorflow_tensor.py +++ b/docarray/typing/tensor/tensorflow_tensor.py @@ -2,23 +2,22 @@ import numpy as np +from docarray.base_doc.base_node import BaseNode +from docarray.typing.proto_register import _register_proto +from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.utils.misc import import_library if TYPE_CHECKING: import tensorflow as tf # type: ignore + from pydantic import BaseConfig + from pydantic.fields import ModelField + + from docarray.computation.tensorflow_backend import TensorFlowCompBackend + from docarray.proto import NdArrayProto else: tf = import_library('tensorflow', raise_error=True) -from docarray.typing.proto_register import _register_proto -from docarray.typing.tensor.abstract_tensor import AbstractTensor - -if TYPE_CHECKING: - from pydantic.fields import ModelField - from pydantic import BaseConfig - from docarray.proto import NdArrayProto - from docarray.computation.tensorflow_backend import TensorFlowCompBackend -from docarray.base_doc.base_node import BaseNode T = TypeVar('T', bound='TensorFlowTensor') ShapeT = TypeVar('ShapeT') diff --git a/docarray/typing/tensor/torch_tensor.py b/docarray/typing/tensor/torch_tensor.py index a4c9e80a850..ed0e7f9ea29 100644 --- a/docarray/typing/tensor/torch_tensor.py +++ b/docarray/typing/tensor/torch_tensor.py @@ -3,23 +3,21 @@ import numpy as np +from docarray.base_doc.base_node import BaseNode from docarray.typing.proto_register import _register_proto from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.utils.misc import import_library if TYPE_CHECKING: + import torch from pydantic import BaseConfig from pydantic.fields import ModelField from docarray.computation.torch_backend import TorchCompBackend from docarray.proto import NdArrayProto - -if TYPE_CHECKING: - import torch else: torch = import_library('torch', raise_error=True) -from docarray.base_doc.base_node import BaseNode T = TypeVar('T', bound='TorchTensor') ShapeT = TypeVar('ShapeT') diff --git a/docarray/utils/_internal/misc.py b/docarray/utils/_internal/misc.py index f7af860082b..9943541e5c0 100644 --- a/docarray/utils/_internal/misc.py +++ b/docarray/utils/_internal/misc.py @@ -1,5 +1,6 @@ import importlib -from typing import Any +import types +from typing import Any, Optional import numpy as np @@ -18,8 +19,6 @@ else: tf_imported = True -import types -from typing import Optional INSTALL_INSTRUCTIONS = { 'pandas': 'pip install "docarray[common]"', @@ -33,6 +32,8 @@ 'fastapi': 'pip install "docarray[web]"', 'torch': 'pip install "docarray[torch]"', 'tensorflow': 'pip install pip install protobuf==3.19.0 tensorflow', + 'smart-open.open': 'pip install "docarray[aws]"', + 'hubble': 'pip install "docarray[jac]"', } From 9d148360d26279b1f1f4c1022033c282c3522bf4 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 15:29:44 +0200 Subject: [PATCH 22/66] fix: tf import in doc index Signed-off-by: anna-charlotte --- docarray/index/backends/hnswlib.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 4e5fa1db682..220c55cf138 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -40,10 +40,11 @@ from docarray.typing import TensorFlowTensor else: - hnswlib = import_library('hnswlib', raise_error=True) + hnswlib = import_library('hnswlib', raise_error=False) torch = import_library('torch', raise_error=False) tf = import_library('tensorflow', raise_error=False) - from docarray.typing import TensorFlowTensor + if tf is not None: + from docarray.typing import TensorFlowTensor HNSWLIB_PY_VEC_TYPES = [list, tuple, np.ndarray] From 843786645205748dd8970e30d06ed3429903605f Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 18:26:49 +0200 Subject: [PATCH 23/66] fix: add getattr on module level Signed-off-by: anna-charlotte --- docarray/base_doc/__init__.py | 14 +++- docarray/base_doc/doc_response.py | 12 +--- docarray/index/__init__.py | 12 +++- docarray/store/__init__.py | 23 ++++++- docarray/store/jac.py | 14 ++-- docarray/store/s3.py | 15 +++-- docarray/typing/__init__.py | 68 +++++++++++--------- docarray/typing/tensor/__init__.py | 35 +++++----- docarray/typing/tensor/audio/__init__.py | 32 +++++---- docarray/typing/tensor/embedding/__init__.py | 27 ++++---- docarray/typing/tensor/image/__init__.py | 29 +++++---- docarray/typing/tensor/torch_tensor.py | 6 +- docarray/typing/tensor/video/__init__.py | 32 +++++---- docarray/utils/_internal/misc.py | 30 +++++---- 14 files changed, 213 insertions(+), 136 deletions(-) diff --git a/docarray/base_doc/__init__.py b/docarray/base_doc/__init__.py index 5fdeed1a807..9e145403066 100644 --- a/docarray/base_doc/__init__.py +++ b/docarray/base_doc/__init__.py @@ -1,6 +1,16 @@ from docarray.base_doc.any_doc import AnyDoc from docarray.base_doc.base_node import BaseNode from docarray.base_doc.doc import BaseDoc -from docarray.base_doc.doc_response import DocResponse -__all__ = ['AnyDoc', 'BaseDoc', 'BaseNode', 'DocResponse'] +__all__ = ['AnyDoc', 'BaseDoc', 'BaseNode'] + +from docarray.utils.misc import import_library + + +def __getattr__(name: str): + if name == 'DocResponse': + import_library('fastapi', raise_error=True) + from docarray.base_doc.doc_response import DocResponse + + __all__.extend(['DocResponse']) + return DocResponse diff --git a/docarray/base_doc/doc_response.py b/docarray/base_doc/doc_response.py index c960cf6da4b..f0a1686a4a8 100644 --- a/docarray/base_doc/doc_response.py +++ b/docarray/base_doc/doc_response.py @@ -5,16 +5,8 @@ if TYPE_CHECKING: from fastapi.responses import JSONResponse else: - fastapi = import_library('fastapi', raise_error=False) - if fastapi is None: - - class NoImportResponse: - def __init__(self, *args, **kwargs): - ImportError('fastapi is not installed') - - JSONResponse = NoImportResponse # type: ignore - else: - from fastapi.responses import JSONResponse + fastapi = import_library('fastapi', raise_error=True) + JSONResponse = fastapi.responses.JSONResponse class DocResponse(JSONResponse): diff --git a/docarray/index/__init__.py b/docarray/index/__init__.py index 5fdbf8ad736..a4822c10be5 100644 --- a/docarray/index/__init__.py +++ b/docarray/index/__init__.py @@ -1,3 +1,11 @@ -from docarray.index.backends.hnswlib import HnswDocumentIndex +from docarray.utils.misc import import_library -__all__ = ['HnswDocumentIndex'] +__all__ = [] + + +def __getattr__(name: str): + if name == 'HnswDocumentIndex': + import_library('hnswlib', raise_error=True) + from docarray.index.backends.hnswlib import HnswDocumentIndex # noqa: F401 + + __all__.extend(['HnswDocumentIndex']) diff --git a/docarray/store/__init__.py b/docarray/store/__init__.py index 20eb3af1a51..63d58870037 100644 --- a/docarray/store/__init__.py +++ b/docarray/store/__init__.py @@ -1,5 +1,22 @@ from docarray.store.file import FileDocStore -from docarray.store.jac import JACDocStore -from docarray.store.s3 import S3DocStore -__all__ = ['JACDocStore', 'FileDocStore', 'S3DocStore'] +__all__ = ['FileDocStore'] + + +from docarray.utils.misc import import_library + + +def __getattr__(name: str): + if name == 'JACDocStore': + import_library('hubble', raise_error=True) + from docarray.store.jac import JACDocStore # noqa: F401 + + __all__.extend(['JACDocStore']) + + elif name == 'S3DocStore': + import_library('smart_open', raise_error=True) + import_library('botocore', raise_error=True) + import_library('boto3', raise_error=True) + from docarray.store.s3 import S3DocStore # noqa: F401 + + __all__.extend(['S3DocStore']) diff --git a/docarray/store/jac.py b/docarray/store/jac.py index c9350fae18a..b2b2564a91e 100644 --- a/docarray/store/jac.py +++ b/docarray/store/jac.py @@ -14,10 +14,6 @@ Union, ) -import hubble -from hubble import Client as HubbleClient -from hubble.client.endpoints import EndpointsV2 - from docarray.store.abstract_doc_store import AbstractDocStore from docarray.store.helpers import ( _BufferedCachingRequestReader, @@ -25,12 +21,22 @@ raise_req_error, ) from docarray.utils._internal.cache import _get_cache_path +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: # pragma: no cover import io from docarray import BaseDoc, DocArray +if TYPE_CHECKING: + import hubble + from hubble import Client as HubbleClient + from hubble.client.endpoints import EndpointsV2 +else: + hubble = import_library('hubble', raise_error=True) + HubbleClient = hubble.Client + EndpointsV2 = hubble.client.endpoints.EndpointsV2 + def _get_length_from_summary(summary: List[Dict]) -> Optional[int]: """Get the length from summary.""" diff --git a/docarray/store/s3.py b/docarray/store/s3.py index 7711432900c..23534d556fd 100644 --- a/docarray/store/s3.py +++ b/docarray/store/s3.py @@ -1,19 +1,22 @@ import io import logging from pathlib import Path -from typing import Dict, Iterator, List, Optional, Type, TypeVar - -import boto3 -import botocore -from smart_open import open -from typing_extensions import TYPE_CHECKING +from typing import TYPE_CHECKING, Dict, Iterator, List, Optional, Type, TypeVar from docarray.store.abstract_doc_store import AbstractDocStore from docarray.store.helpers import _from_binary_stream, _to_binary_stream from docarray.utils._internal.cache import _get_cache_path +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: # pragma: no cover from docarray import BaseDoc, DocArray + import boto3 + import botocore + from smart_open import open +else: + open = import_library('smart_open', raise_error=True).open + boto3 = import_library('boto3', raise_error=True) + botocore = import_library('botocore', raise_error=True) SelfS3DocStore = TypeVar('SelfS3DocStore', bound='S3DocStore') diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index 3deefe96f01..490bc6469c5 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -37,39 +37,43 @@ 'ImageNdArray', ] -from docarray.utils._internal.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import import_library -torch_available = is_torch_available() -if torch_available: - from docarray.typing.tensor import TorchEmbedding, TorchTensor # noqa: F401 - from docarray.typing.tensor.audio.audio_torch_tensor import AudioTorchTensor # noqa - from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 - from docarray.typing.tensor.video.video_torch_tensor import VideoTorchTensor # noqa +torch_tensors = [ + 'AudioTorchTensor', + 'TorchEmbedding', + 'TorchTensor', + 'VideoTorchTensor', + 'ImageTorchTensor', +] + +tf_tensors = [ + 'TensorFlowTensor', + 'TensorFlowEmbedding', + 'AudioTensorFlowTensor', + 'ImageTensorFlowTensor', + 'VideoTensorFlowTensor', +] + + +def __getattr__(name: str): + if name in torch_tensors: + import_library('torch', raise_error=True) + + from docarray.typing.tensor import TorchEmbedding, TorchTensor # noqa: F401 + from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F401 + from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 + from docarray.typing.tensor.video import VideoTorchTensor # noqa: F401 + + __all__.extend(torch_tensors) - __all__.extend( - [ - 'AudioTorchTensor', - 'TorchEmbedding', - 'TorchTensor', - 'VideoTorchTensor', - 'ImageTorchTensor', - ] - ) + elif name in tf_tensors: + import_library('tensorflow', raise_error=True) -tf_available = is_tf_available() -if tf_available: - from docarray.typing.tensor import TensorFlowTensor # noqa: F401 - from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 - from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa + from docarray.typing.tensor import TensorFlowTensor # noqa: F401 + from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 + from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa - __all__.extend( - [ - 'TensorFlowTensor', - 'TensorFlowEmbedding', - 'AudioTensorFlowTensor', - 'ImageTensorFlowTensor', - 'VideoTensorFlowTensor', - ] - ) + __all__.extend(tf_tensors) diff --git a/docarray/typing/tensor/__init__.py b/docarray/typing/tensor/__init__.py index f2be5ecc0ac..5c2773b2bf6 100644 --- a/docarray/typing/tensor/__init__.py +++ b/docarray/typing/tensor/__init__.py @@ -10,26 +10,31 @@ 'NdArrayEmbedding', 'ImageNdArray', 'ImageTensor', - 'TensorFlowTensor', ] -from docarray.utils._internal.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import import_library -torch_available = is_torch_available() -if torch_available: - from docarray.typing.tensor.embedding import TorchEmbedding # noqa: F401 - from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 - from docarray.typing.tensor.torch_tensor import TorchTensor # noqa: F401 +torch_tensors = ['TorchEmbedding', 'ImageTorchTensor', 'TorchTensor'] +tf_tensors = ['TensorFlowEmbedding', 'TensorFlowTensor', 'ImageTensorFlowTensor'] - __all__.extend(['TorchEmbedding', 'TorchTensor', 'ImageTorchTensor']) - torch_available = is_torch_available() +def __getattr__(name: str): + if name in torch_tensors: + import_library('torch', raise_error=True) + from docarray.typing.tensor.embedding import TorchEmbedding # noqa: F401 + from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 + from docarray.typing.tensor.torch_tensor import TorchTensor # noqa: F401 -tf_available = is_tf_available() -if tf_available: - from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 - from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.tensorflow_tensor import TensorFlowTensor # noqa: F401 + __all__.extend(torch_tensors) - __all__.extend(['TensorFlowEmbedding', 'TensorFlowTensor', 'ImageTensorFlowTensor']) + elif name in tf_tensors: + import_library('tensorflow', raise_error=True) + + from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 + from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.tensorflow_tensor import ( # noqa: F401 + TensorFlowTensor, + ) + + __all__.extend(tf_tensors) diff --git a/docarray/typing/tensor/audio/__init__.py b/docarray/typing/tensor/audio/__init__.py index 4f730451603..ebca0a80096 100644 --- a/docarray/typing/tensor/audio/__init__.py +++ b/docarray/typing/tensor/audio/__init__.py @@ -1,20 +1,28 @@ from docarray.typing.tensor.audio.audio_ndarray import AudioNdArray +from docarray.typing.tensor.audio.audio_tensor import AudioTensor -__all__ = ['AudioNdArray'] +__all__ = ['AudioNdArray', 'AudioTensor'] -from docarray.utils._internal.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import import_library -torch_available = is_torch_available() -if torch_available: - from docarray.typing.tensor.audio.audio_torch_tensor import AudioTorchTensor # noqa +torch_tensors = ['AudioTorchTensor'] +tf_tensors = ['AudioTensorFlowTensor'] - __all__.extend(['AudioTorchTensor']) +def __getattr__(name: str): + if name in torch_tensors: + import_library('torch', raise_error=True) + from docarray.typing.tensor.audio.audio_torch_tensor import ( # noqa + AudioTorchTensor, + ) -tf_available = is_tf_available() -if tf_available: - from docarray.typing.tensor.audio.audio_tensorflow_tensor import ( # noqa - AudioTensorFlowTensor, - ) + __all__.extend(['AudioTorchTensor']) - __all__.extend(['AudioTensorFlowTensor']) + elif name in tf_tensors: + import_library('tensorflow', raise_error=True) + + from docarray.typing.tensor.audio.audio_tensorflow_tensor import ( # noqa + AudioTensorFlowTensor, + ) + + __all__.extend(tf_tensors) diff --git a/docarray/typing/tensor/embedding/__init__.py b/docarray/typing/tensor/embedding/__init__.py index f844b7abaa9..e70f2ef1b07 100644 --- a/docarray/typing/tensor/embedding/__init__.py +++ b/docarray/typing/tensor/embedding/__init__.py @@ -3,19 +3,24 @@ __all__ = ['NdArrayEmbedding', 'AnyEmbedding'] -from docarray.utils._internal.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import import_library -torch_available = is_torch_available() -if torch_available: - from docarray.typing.tensor.embedding.torch import TorchEmbedding # noqa F401 +torch_tensors = ['TorchEmbedding'] +tf_tensors = ['TensorFlowEmbedding'] - __all__.append('TorchEmbedding') +def __getattr__(name: str): + if name in torch_tensors: + import_library('torch', raise_error=True) + from docarray.typing.tensor.embedding.torch import TorchEmbedding # noqa -tf_available = is_tf_available() -if tf_available: - from docarray.typing.tensor.embedding.tensorflow import ( # noqa F401 - TensorFlowEmbedding, - ) + __all__.extend(torch_tensors) - __all__.append('TensorFlowEmbedding') + elif name in tf_tensors: + import_library('tensorflow', raise_error=True) + + from docarray.typing.tensor.embedding.tensorflow import ( # noqa + TensorFlowEmbedding, + ) + + __all__.extend(tf_tensors) diff --git a/docarray/typing/tensor/image/__init__.py b/docarray/typing/tensor/image/__init__.py index 2fb32d452ab..1f6701145bd 100644 --- a/docarray/typing/tensor/image/__init__.py +++ b/docarray/typing/tensor/image/__init__.py @@ -3,19 +3,26 @@ __all__ = ['ImageNdArray', 'ImageTensor'] -from docarray.utils._internal.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import import_library -torch_available = is_torch_available() -if torch_available: - from docarray.typing.tensor.image.image_torch_tensor import ImageTorchTensor # noqa +torch_tensors = ['ImageTorchTensor'] +tf_tensors = ['ImageTensorFlowTensor'] - __all__.extend(['ImageTorchTensor']) +def __getattr__(name: str): + if name in torch_tensors: + import_library('torch', raise_error=True) + from docarray.typing.tensor.image.image_torch_tensor import ( # noqa + ImageTorchTensor, + ) -tf_available = is_tf_available() -if tf_available: - from docarray.typing.tensor.image.image_tensorflow_tensor import ( # noqa - ImageTensorFlowTensor, - ) + __all__.extend(torch_tensors) - __all__.extend(['ImageTensorFlowTensor']) + elif name in tf_tensors: + import_library('tensorflow', raise_error=True) + + from docarray.typing.tensor.image.image_tensorflow_tensor import ( # noqa + ImageTensorFlowTensor, + ) + + __all__.extend(tf_tensors) diff --git a/docarray/typing/tensor/torch_tensor.py b/docarray/typing/tensor/torch_tensor.py index ed0e7f9ea29..b7571ebefc8 100644 --- a/docarray/typing/tensor/torch_tensor.py +++ b/docarray/typing/tensor/torch_tensor.py @@ -9,7 +9,6 @@ from docarray.utils.misc import import_library if TYPE_CHECKING: - import torch from pydantic import BaseConfig from pydantic.fields import ModelField @@ -38,7 +37,10 @@ class metaTorchAndNode( @_register_proto(proto_type_name='torch_tensor') class TorchTensor( - torch.Tensor, AbstractTensor, Generic[ShapeT], metaclass=metaTorchAndNode + torch.Tensor, + AbstractTensor, + Generic[ShapeT], + metaclass=metaTorchAndNode, ): # Subclassing torch.Tensor following the advice from here: # https://pytorch.org/docs/stable/notes/extending.html#subclassing-torch-tensor diff --git a/docarray/typing/tensor/video/__init__.py b/docarray/typing/tensor/video/__init__.py index 424758cfc91..1c1ebe743ca 100644 --- a/docarray/typing/tensor/video/__init__.py +++ b/docarray/typing/tensor/video/__init__.py @@ -1,20 +1,28 @@ from docarray.typing.tensor.video.video_ndarray import VideoNdArray +from docarray.typing.tensor.video.video_tensor import VideoTensor -__all__ = ['VideoNdArray'] +__all__ = ['VideoNdArray', 'VideoTensor'] -from docarray.utils._internal.misc import is_tf_available, is_torch_available +from docarray.utils._internal.misc import import_library -torch_available = is_torch_available() -if torch_available: - from docarray.typing.tensor.video.video_torch_tensor import VideoTorchTensor # noqa +torch_tensors = ['VideoTorchTensor'] +tf_tensors = ['VideoTensorFlowTensor'] - __all__.extend(['VideoTorchTensor']) +def __getattr__(name: str): + if name in torch_tensors: + import_library('torch', raise_error=True) + from docarray.typing.tensor.video.video_torch_tensor import ( # noqa + VideoTorchTensor, + ) -tf_available = is_tf_available() -if tf_available: - from docarray.typing.tensor.video.video_tensorflow_tensor import ( # noqa: F401 - VideoTensorFlowTensor, - ) + __all__.extend(torch_tensors) - __all__.extend(['VideoTensorFlowTensor']) + elif name in tf_tensors: + import_library('tensorflow', raise_error=True) + + from docarray.typing.tensor.video.video_tensorflow_tensor import ( # noqa + VideoTensorFlowTensor, + ) + + __all__.extend(tf_tensors) diff --git a/docarray/utils/_internal/misc.py b/docarray/utils/_internal/misc.py index 9943541e5c0..0c7da5b0d86 100644 --- a/docarray/utils/_internal/misc.py +++ b/docarray/utils/_internal/misc.py @@ -21,19 +21,21 @@ INSTALL_INSTRUCTIONS = { - 'pandas': 'pip install "docarray[common]"', - 'google.protobuf': 'pip install "docarray[common]"', - 'lz4.frame': 'pip install "docarray[common]"', - 'PIL.Image': 'pip install "docarray[image]"', - 'pydub': 'pip install "docarray[audio]"', - 'av': 'pip install "docarray[video]"', - 'trimesh': 'pip install "docarray[mesh]"', - 'hnswlib': 'pip install "docarray[hnswlib]"', - 'fastapi': 'pip install "docarray[web]"', - 'torch': 'pip install "docarray[torch]"', - 'tensorflow': 'pip install pip install protobuf==3.19.0 tensorflow', - 'smart-open.open': 'pip install "docarray[aws]"', - 'hubble': 'pip install "docarray[jac]"', + 'pandas': '"docarray[common]"', + 'google.protobuf': '"docarray[common]"', + 'lz4.frame': '"docarray[common]"', + 'PIL.Image': '"docarray[image]"', + 'pydub': '"docarray[audio]"', + 'av': '"docarray[video]"', + 'trimesh': '"docarray[mesh]"', + 'hnswlib': '"docarray[hnswlib]"', + 'fastapi': '"docarray[web]"', + 'torch': '"docarray[torch]"', + 'tensorflow': 'protobuf==3.19.0 tensorflow', + 'smart_open': '"docarray[aws]"', + 'boto3': '"docarray[aws]"', + 'botocore': '"docarray[aws]"', + 'hubble': '"docarray[jac]"', } @@ -49,7 +51,7 @@ def import_library( if lib is None and raise_error: raise RuntimeError( f'{package} is not installed. To install all the necessary libraries to use the hnsw backend, ' - f'please do: `{INSTALL_INSTRUCTIONS[package]}`.' + f'please do: `pip install {INSTALL_INSTRUCTIONS[package]}`.' ) else: return lib From d26dae404e4c5781fbb8efa45dfa072b6b6a0d8e Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 18:33:30 +0200 Subject: [PATCH 24/66] fix: import torch for type checking Signed-off-by: anna-charlotte --- docarray/typing/tensor/torch_tensor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docarray/typing/tensor/torch_tensor.py b/docarray/typing/tensor/torch_tensor.py index b7571ebefc8..eb62cff2895 100644 --- a/docarray/typing/tensor/torch_tensor.py +++ b/docarray/typing/tensor/torch_tensor.py @@ -9,6 +9,7 @@ from docarray.utils.misc import import_library if TYPE_CHECKING: + import torch from pydantic import BaseConfig from pydantic.fields import ModelField From 0e868f3c3e9d1c92404303c5d8a28d1ea9218740 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 19:14:18 +0200 Subject: [PATCH 25/66] fix: add type checking Signed-off-by: anna-charlotte --- docarray/base_document/__init__.py | 20 ++++++++ docarray/index/__init__.py | 10 +++- docarray/store/__init__.py | 16 ++++-- docarray/typing/__init__.py | 54 ++++++++++++++++---- docarray/typing/tensor/__init__.py | 38 +++++++++++--- docarray/typing/tensor/audio/__init__.py | 19 ++++--- docarray/typing/tensor/embedding/__init__.py | 17 +++--- docarray/typing/tensor/image/__init__.py | 19 ++++--- docarray/typing/tensor/video/__init__.py | 19 ++++--- 9 files changed, 165 insertions(+), 47 deletions(-) create mode 100644 docarray/base_document/__init__.py diff --git a/docarray/base_document/__init__.py b/docarray/base_document/__init__.py new file mode 100644 index 00000000000..6001defaaa9 --- /dev/null +++ b/docarray/base_document/__init__.py @@ -0,0 +1,20 @@ +from typing_extensions import TYPE_CHECKING + +from docarray.base_document.any_document import AnyDocument +from docarray.base_document.base_node import BaseNode +from docarray.base_document.document import BaseDocument + +__all__ = ['AnyDocument', 'BaseDocument', 'BaseNode'] + +from docarray.utils.misc import import_library + +if TYPE_CHECKING: + from docarray.base_document.document_response import DocumentResponse # noqa: F401 + + +def __getattr__(name: str): + if name == 'DocumentResponse': + import_library('fastapi', raise_error=True) + from docarray.base_document.document_response import DocumentResponse # noqa + + __all__.extend(['DocumentResponse']) diff --git a/docarray/index/__init__.py b/docarray/index/__init__.py index a4822c10be5..1f6f8af8aca 100644 --- a/docarray/index/__init__.py +++ b/docarray/index/__init__.py @@ -1,11 +1,17 @@ +from typing import TYPE_CHECKING + from docarray.utils.misc import import_library __all__ = [] +if TYPE_CHECKING: + from docarray.index.backends.hnswlib import HnswDocumentIndex # noqa: F401 + def __getattr__(name: str): if name == 'HnswDocumentIndex': import_library('hnswlib', raise_error=True) - from docarray.index.backends.hnswlib import HnswDocumentIndex # noqa: F401 + from docarray.index.backends.hnswlib import HnswDocumentIndex # noqa - __all__.extend(['HnswDocumentIndex']) + __all__.append('HnswDocumentIndex') + return HnswDocumentIndex diff --git a/docarray/store/__init__.py b/docarray/store/__init__.py index 63d58870037..1456c9986ab 100644 --- a/docarray/store/__init__.py +++ b/docarray/store/__init__.py @@ -1,3 +1,5 @@ +from typing import TYPE_CHECKING + from docarray.store.file import FileDocStore __all__ = ['FileDocStore'] @@ -5,18 +7,24 @@ from docarray.utils.misc import import_library +if TYPE_CHECKING: + from docarray.store.jac import JACDocStore # noqa: F401 + from docarray.store.s3 import S3DocStore # noqa: F401 + def __getattr__(name: str): if name == 'JACDocStore': import_library('hubble', raise_error=True) - from docarray.store.jac import JACDocStore # noqa: F401 + from docarray.store.jac import JACDocStore # noqa - __all__.extend(['JACDocStore']) + __all__.append('JACDocStore') + return JACDocStore elif name == 'S3DocStore': import_library('smart_open', raise_error=True) import_library('botocore', raise_error=True) import_library('boto3', raise_error=True) - from docarray.store.s3 import S3DocStore # noqa: F401 + from docarray.store.s3 import S3DocStore # noqa - __all__.extend(['S3DocStore']) + __all__.append('S3DocStore') + return S3DocStore diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index 490bc6469c5..1db60181415 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -1,3 +1,5 @@ +from typing_extensions import TYPE_CHECKING + from docarray.typing.bytes import ImageBytes from docarray.typing.id import ID from docarray.typing.tensor import ImageNdArray, ImageTensor @@ -39,6 +41,18 @@ from docarray.utils._internal.misc import import_library +if TYPE_CHECKING: + from docarray.typing.tensor import TensorFlowTensor # noqa: F401 + from docarray.typing.tensor import TorchEmbedding, TorchTensor # noqa: F401 + from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F401 + from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 + from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 + from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.video import VideoTorchTensor # noqa: F401 + + torch_tensors = [ 'AudioTorchTensor', 'TorchEmbedding', @@ -60,20 +74,42 @@ def __getattr__(name: str): if name in torch_tensors: import_library('torch', raise_error=True) - from docarray.typing.tensor import TorchEmbedding, TorchTensor # noqa: F401 - from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F401 - from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 - from docarray.typing.tensor.video import VideoTorchTensor # noqa: F401 + from docarray.typing.tensor import TorchEmbedding, TorchTensor # noqa: F811 + from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F811 + from docarray.typing.tensor.image import ImageTorchTensor # noqa: F811 + from docarray.typing.tensor.video import VideoTorchTensor # noqa: F811 __all__.extend(torch_tensors) + if name == 'TorchTensor': + return TorchTensor + elif name == 'TorchEmbedding': + return TorchEmbedding + elif name == 'AudioTorchTensor': + return AudioTorchTensor + elif name == 'ImageTorchTensor': + return ImageTorchTensor + elif name == 'VideoTorchTensor': + return VideoTorchTensor + elif name in tf_tensors: import_library('tensorflow', raise_error=True) - from docarray.typing.tensor import TensorFlowTensor # noqa: F401 - from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 - from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 + from docarray.typing.tensor import TensorFlowTensor # noqa + from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa + from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa + from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa - __all__.extend(tf_tensors) + __all__.extend(torch_tensors) + + if name == 'TensorFlowTensor': + return TensorFlowTensor + elif name == 'TensorFlowEmbedding': + return TensorFlowEmbedding + elif name == 'AudioTensorFlowTensor': + return AudioTensorFlowTensor + elif name == 'ImageTensorFlowTensor': + return ImageTensorFlowTensor + elif name == 'VideoTensorFlowTensor': + return VideoTensorFlowTensor diff --git a/docarray/typing/tensor/__init__.py b/docarray/typing/tensor/__init__.py index 5c2773b2bf6..c40feac93b8 100644 --- a/docarray/typing/tensor/__init__.py +++ b/docarray/typing/tensor/__init__.py @@ -1,3 +1,5 @@ +from typing_extensions import TYPE_CHECKING + from docarray.typing.tensor.embedding import AnyEmbedding, NdArrayEmbedding from docarray.typing.tensor.image import ImageNdArray, ImageTensor from docarray.typing.tensor.ndarray import NdArray @@ -14,6 +16,14 @@ from docarray.utils._internal.misc import import_library +if TYPE_CHECKING: + from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 + from docarray.typing.tensor.embedding import TorchEmbedding # noqa: F401 + from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 + from docarray.typing.tensor.tensorflow_tensor import TensorFlowTensor # noqa: F401 + from docarray.typing.tensor.torch_tensor import TorchTensor # noqa: F401 + torch_tensors = ['TorchEmbedding', 'ImageTorchTensor', 'TorchTensor'] tf_tensors = ['TensorFlowEmbedding', 'TensorFlowTensor', 'ImageTensorFlowTensor'] @@ -22,19 +32,31 @@ def __getattr__(name: str): if name in torch_tensors: import_library('torch', raise_error=True) - from docarray.typing.tensor.embedding import TorchEmbedding # noqa: F401 - from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 - from docarray.typing.tensor.torch_tensor import TorchTensor # noqa: F401 + from docarray.typing.tensor.embedding import TorchEmbedding # noqa + from docarray.typing.tensor.image import ImageTorchTensor # noqa + from docarray.typing.tensor.torch_tensor import TorchTensor # noqa __all__.extend(torch_tensors) + if name == 'TorchTensor': + return TorchTensor + elif name == 'TorchEmbedding': + return TorchEmbedding + elif name == 'ImageTorchTensor': + return ImageTorchTensor + elif name in tf_tensors: import_library('tensorflow', raise_error=True) - from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 - from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.tensorflow_tensor import ( # noqa: F401 - TensorFlowTensor, - ) + from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa + from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa + from docarray.typing.tensor.tensorflow_tensor import TensorFlowTensor # noqa __all__.extend(tf_tensors) + + if name == 'TensorFlowTensor': + return TensorFlowTensor + elif name == 'TensorFlowEmbedding': + return TensorFlowEmbedding + elif name == 'ImageTensorFlowTensor': + return ImageTensorFlowTensor diff --git a/docarray/typing/tensor/audio/__init__.py b/docarray/typing/tensor/audio/__init__.py index ebca0a80096..30973a7ad12 100644 --- a/docarray/typing/tensor/audio/__init__.py +++ b/docarray/typing/tensor/audio/__init__.py @@ -1,3 +1,5 @@ +from typing import TYPE_CHECKING + from docarray.typing.tensor.audio.audio_ndarray import AudioNdArray from docarray.typing.tensor.audio.audio_tensor import AudioTensor @@ -5,24 +7,29 @@ from docarray.utils._internal.misc import import_library -torch_tensors = ['AudioTorchTensor'] -tf_tensors = ['AudioTensorFlowTensor'] +if TYPE_CHECKING: + from docarray.typing.tensor.audio.audio_tensorflow_tensor import ( # noqa + AudioTensorFlowTensor, + ) + from docarray.typing.tensor.audio.audio_torch_tensor import AudioTorchTensor # noqa def __getattr__(name: str): - if name in torch_tensors: + if name == 'AudioTorchTensor': import_library('torch', raise_error=True) from docarray.typing.tensor.audio.audio_torch_tensor import ( # noqa AudioTorchTensor, ) - __all__.extend(['AudioTorchTensor']) + __all__.append('AudioTorchTensor') + return AudioTorchTensor - elif name in tf_tensors: + elif name == 'AudioTensorFlowTensor': import_library('tensorflow', raise_error=True) from docarray.typing.tensor.audio.audio_tensorflow_tensor import ( # noqa AudioTensorFlowTensor, ) - __all__.extend(tf_tensors) + __all__.append('AudioTensorFlowTensor') + return AudioTensorFlowTensor diff --git a/docarray/typing/tensor/embedding/__init__.py b/docarray/typing/tensor/embedding/__init__.py index e70f2ef1b07..42814851999 100644 --- a/docarray/typing/tensor/embedding/__init__.py +++ b/docarray/typing/tensor/embedding/__init__.py @@ -1,3 +1,5 @@ +from typing_extensions import TYPE_CHECKING + from docarray.typing.tensor.embedding.embedding import AnyEmbedding from docarray.typing.tensor.embedding.ndarray import NdArrayEmbedding @@ -5,22 +7,25 @@ from docarray.utils._internal.misc import import_library -torch_tensors = ['TorchEmbedding'] -tf_tensors = ['TensorFlowEmbedding'] +if TYPE_CHECKING: + from docarray.typing.tensor.embedding.tensorflow import TensorFlowEmbedding # noqa + from docarray.typing.tensor.embedding.torch import TorchEmbedding # noqa def __getattr__(name: str): - if name in torch_tensors: + if name == 'TorchEmbedding': import_library('torch', raise_error=True) from docarray.typing.tensor.embedding.torch import TorchEmbedding # noqa - __all__.extend(torch_tensors) + __all__.append('TorchEmbedding') + return TorchEmbedding - elif name in tf_tensors: + elif name == 'TensorFlowEmbedding': import_library('tensorflow', raise_error=True) from docarray.typing.tensor.embedding.tensorflow import ( # noqa TensorFlowEmbedding, ) - __all__.extend(tf_tensors) + __all__.append('TensorFlowEmbedding') + return TensorFlowEmbedding diff --git a/docarray/typing/tensor/image/__init__.py b/docarray/typing/tensor/image/__init__.py index 1f6701145bd..1c199a5ff81 100644 --- a/docarray/typing/tensor/image/__init__.py +++ b/docarray/typing/tensor/image/__init__.py @@ -1,3 +1,5 @@ +from typing_extensions import TYPE_CHECKING + from docarray.typing.tensor.image.image_ndarray import ImageNdArray from docarray.typing.tensor.image.image_tensor import ImageTensor @@ -5,24 +7,29 @@ from docarray.utils._internal.misc import import_library -torch_tensors = ['ImageTorchTensor'] -tf_tensors = ['ImageTensorFlowTensor'] +if TYPE_CHECKING: + from docarray.typing.tensor.image.image_tensorflow_tensor import ( # noqa + ImageTensorFlowTensor, + ) + from docarray.typing.tensor.image.image_torch_tensor import ImageTorchTensor # noqa def __getattr__(name: str): - if name in torch_tensors: + if name == 'ImageTorchTensor': import_library('torch', raise_error=True) from docarray.typing.tensor.image.image_torch_tensor import ( # noqa ImageTorchTensor, ) - __all__.extend(torch_tensors) + __all__.append('ImageTorchTensor') + return ImageTorchTensor - elif name in tf_tensors: + elif name == 'ImageTensorFlowTensor': import_library('tensorflow', raise_error=True) from docarray.typing.tensor.image.image_tensorflow_tensor import ( # noqa ImageTensorFlowTensor, ) - __all__.extend(tf_tensors) + __all__.append('ImageTensorFlowTensor') + return ImageTensorFlowTensor diff --git a/docarray/typing/tensor/video/__init__.py b/docarray/typing/tensor/video/__init__.py index 1c1ebe743ca..8262e30f357 100644 --- a/docarray/typing/tensor/video/__init__.py +++ b/docarray/typing/tensor/video/__init__.py @@ -1,3 +1,5 @@ +from typing_extensions import TYPE_CHECKING + from docarray.typing.tensor.video.video_ndarray import VideoNdArray from docarray.typing.tensor.video.video_tensor import VideoTensor @@ -5,24 +7,29 @@ from docarray.utils._internal.misc import import_library -torch_tensors = ['VideoTorchTensor'] -tf_tensors = ['VideoTensorFlowTensor'] +if TYPE_CHECKING: + from docarray.typing.tensor.video.video_tensorflow_tensor import ( # noqa + VideoTensorFlowTensor, + ) + from docarray.typing.tensor.video.video_torch_tensor import VideoTorchTensor # noqa def __getattr__(name: str): - if name in torch_tensors: + if name == 'VideoTorchTensor': import_library('torch', raise_error=True) from docarray.typing.tensor.video.video_torch_tensor import ( # noqa VideoTorchTensor, ) - __all__.extend(torch_tensors) + __all__.append('VideoTorchTensor') + return VideoTorchTensor - elif name in tf_tensors: + elif name == 'VideoTensorFlowTensor': import_library('tensorflow', raise_error=True) from docarray.typing.tensor.video.video_tensorflow_tensor import ( # noqa VideoTensorFlowTensor, ) - __all__.extend(tf_tensors) + __all__.append('VideoTensorFlowTensor') + return VideoTensorFlowTensor From 58b6c529addc963bb5535cfafb5d28defda712f2 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Mon, 27 Mar 2023 19:22:21 +0200 Subject: [PATCH 26/66] fix: test cross backend Signed-off-by: anna-charlotte --- tests/units/typing/tensor/test_cross_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/units/typing/tensor/test_cross_backend.py b/tests/units/typing/tensor/test_cross_backend.py index 702cd678d6f..4e4528c0502 100644 --- a/tests/units/typing/tensor/test_cross_backend.py +++ b/tests/units/typing/tensor/test_cross_backend.py @@ -6,7 +6,7 @@ try: from docarray.typing import TensorFlowTensor -except (ImportError, TypeError): +except RuntimeError: pass From 090382db40d8d9ea4efdb7eb7b6e968afa18377a Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 09:31:06 +0200 Subject: [PATCH 27/66] fix: add missing return statement Signed-off-by: anna-charlotte --- docarray/base_document/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docarray/base_document/__init__.py b/docarray/base_document/__init__.py index 6001defaaa9..72f3a424d4e 100644 --- a/docarray/base_document/__init__.py +++ b/docarray/base_document/__init__.py @@ -3,10 +3,10 @@ from docarray.base_document.any_document import AnyDocument from docarray.base_document.base_node import BaseNode from docarray.base_document.document import BaseDocument +from docarray.utils.misc import import_library __all__ = ['AnyDocument', 'BaseDocument', 'BaseNode'] -from docarray.utils.misc import import_library if TYPE_CHECKING: from docarray.base_document.document_response import DocumentResponse # noqa: F401 @@ -18,3 +18,4 @@ def __getattr__(name: str): from docarray.base_document.document_response import DocumentResponse # noqa __all__.extend(['DocumentResponse']) + return DocumentResponse From 7a6783b623dc1c73302282f5d4d98b9f6ea82e4c Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 09:55:51 +0200 Subject: [PATCH 28/66] fix: clean up Signed-off-by: anna-charlotte --- docarray/base_document/__init__.py | 2 +- docarray/typing/bytes/image_bytes.py | 2 +- docarray/typing/tensor/image/abstract_image_tensor.py | 4 ++-- docarray/utils/_internal/compress.py | 6 +++--- docarray/utils/_internal/misc.py | 6 +++--- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docarray/base_document/__init__.py b/docarray/base_document/__init__.py index 72f3a424d4e..28b3bf9c9dd 100644 --- a/docarray/base_document/__init__.py +++ b/docarray/base_document/__init__.py @@ -17,5 +17,5 @@ def __getattr__(name: str): import_library('fastapi', raise_error=True) from docarray.base_document.document_response import DocumentResponse # noqa - __all__.extend(['DocumentResponse']) + __all__.append('DocumentResponse') return DocumentResponse diff --git a/docarray/typing/bytes/image_bytes.py b/docarray/typing/bytes/image_bytes.py index 8c1e2fa11ec..cd75b02a6b1 100644 --- a/docarray/typing/bytes/image_bytes.py +++ b/docarray/typing/bytes/image_bytes.py @@ -91,7 +91,7 @@ class MyDoc(BaseDoc): if TYPE_CHECKING: from PIL import Image as PILImage else: - PILImage = import_library('PIL.Image') + PILImage = import_library('PIL').Image raw_img = PILImage.open(BytesIO(self)) if width or height: diff --git a/docarray/typing/tensor/image/abstract_image_tensor.py b/docarray/typing/tensor/image/abstract_image_tensor.py index 02cec3e5a17..46bb61ed803 100644 --- a/docarray/typing/tensor/image/abstract_image_tensor.py +++ b/docarray/typing/tensor/image/abstract_image_tensor.py @@ -18,7 +18,7 @@ def to_bytes(self, format: str = 'PNG') -> bytes: if TYPE_CHECKING: from PIL import Image as PILImage else: - PILImage = import_library('PIL.Image') + PILImage = import_library('PIL').Image if format == 'jpg': format = 'jpeg' # unify it to ISO standard @@ -42,7 +42,7 @@ def display(self) -> None: if TYPE_CHECKING: from PIL import Image as PILImage else: - PILImage = import_library('PIL.Image') + PILImage = import_library('PIL').Image np_array = self.get_comp_backend().to_numpy(self) img = PILImage.fromarray(np_array) diff --git a/docarray/utils/_internal/compress.py b/docarray/utils/_internal/compress.py index 7b98f64905d..e71136bb3b4 100644 --- a/docarray/utils/_internal/compress.py +++ b/docarray/utils/_internal/compress.py @@ -8,7 +8,7 @@ def _compress_bytes(data: bytes, algorithm: Optional[str] = None) -> bytes: if TYPE_CHECKING: import lz4.frame as lz4_frame # type: ignore else: - lz4_frame = import_library('lz4.frame') + lz4_frame = import_library('lz4').frame data = lz4_frame.compress(data) elif algorithm == 'bz2': @@ -36,7 +36,7 @@ def _decompress_bytes(data: bytes, algorithm: Optional[str] = None) -> bytes: if TYPE_CHECKING: import lz4.frame as lz4_frame # type: ignore else: - lz4_frame = import_library('lz4.frame') + lz4_frame = import_library('lz4').frame data = lz4_frame.decompress(data) elif algorithm == 'bz2': @@ -64,7 +64,7 @@ def _get_compress_ctx(algorithm: Optional[str] = None) -> Optional[Callable]: if TYPE_CHECKING: import lz4.frame as lz4_frame # type: ignore else: - lz4_frame = import_library('lz4.frame') + lz4_frame = import_library('lz4').frame def _fun(x: IO[bytes]): return lz4_frame.LZ4FrameFile(x, 'wb') diff --git a/docarray/utils/_internal/misc.py b/docarray/utils/_internal/misc.py index 0c7da5b0d86..31e7d53282d 100644 --- a/docarray/utils/_internal/misc.py +++ b/docarray/utils/_internal/misc.py @@ -23,8 +23,8 @@ INSTALL_INSTRUCTIONS = { 'pandas': '"docarray[common]"', 'google.protobuf': '"docarray[common]"', - 'lz4.frame': '"docarray[common]"', - 'PIL.Image': '"docarray[image]"', + 'lz4': '"docarray[common]"', + 'PIL': '"docarray[image]"', 'pydub': '"docarray[audio]"', 'av': '"docarray[video]"', 'trimesh': '"docarray[mesh]"', @@ -32,10 +32,10 @@ 'fastapi': '"docarray[web]"', 'torch': '"docarray[torch]"', 'tensorflow': 'protobuf==3.19.0 tensorflow', + 'hubble': '"docarray[jac]"', 'smart_open': '"docarray[aws]"', 'boto3': '"docarray[aws]"', 'botocore': '"docarray[aws]"', - 'hubble': '"docarray[jac]"', } From f87e3b4360b47ef3090e36799bd105ab3e97a8ea Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 10:22:07 +0200 Subject: [PATCH 29/66] fix: update error message Signed-off-by: anna-charlotte --- docarray/utils/_internal/misc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docarray/utils/_internal/misc.py b/docarray/utils/_internal/misc.py index 31e7d53282d..72601e9f57c 100644 --- a/docarray/utils/_internal/misc.py +++ b/docarray/utils/_internal/misc.py @@ -50,8 +50,8 @@ def import_library( if lib is None and raise_error: raise RuntimeError( - f'{package} is not installed. To install all the necessary libraries to use the hnsw backend, ' - f'please do: `pip install {INSTALL_INSTRUCTIONS[package]}`.' + f'The following required library is not installed: {package} ' + f'To install all necessary libraries, run: `pip install {INSTALL_INSTRUCTIONS[package]}`.' ) else: return lib From 91f8da684d9b069121acc0773929f0a5851c546d Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 10:50:35 +0200 Subject: [PATCH 30/66] fix: remove base document init Signed-off-by: anna-charlotte --- docarray/base_document/__init__.py | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 docarray/base_document/__init__.py diff --git a/docarray/base_document/__init__.py b/docarray/base_document/__init__.py deleted file mode 100644 index 28b3bf9c9dd..00000000000 --- a/docarray/base_document/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -from typing_extensions import TYPE_CHECKING - -from docarray.base_document.any_document import AnyDocument -from docarray.base_document.base_node import BaseNode -from docarray.base_document.document import BaseDocument -from docarray.utils.misc import import_library - -__all__ = ['AnyDocument', 'BaseDocument', 'BaseNode'] - - -if TYPE_CHECKING: - from docarray.base_document.document_response import DocumentResponse # noqa: F401 - - -def __getattr__(name: str): - if name == 'DocumentResponse': - import_library('fastapi', raise_error=True) - from docarray.base_document.document_response import DocumentResponse # noqa - - __all__.append('DocumentResponse') - return DocumentResponse From 96087bb88e311904fdf7dc9a8126174abf0dbee8 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 10:53:09 +0200 Subject: [PATCH 31/66] fix: clean up Signed-off-by: anna-charlotte --- docarray/array/array/io.py | 2 +- docarray/documents/mesh/vertices_and_faces.py | 2 +- docarray/documents/point_cloud/points_and_colors.py | 2 +- docarray/proto/__init__.py | 2 +- docarray/typing/bytes/audio_bytes.py | 2 +- docarray/typing/bytes/image_bytes.py | 2 +- docarray/typing/tensor/audio/abstract_audio_tensor.py | 2 +- docarray/typing/tensor/image/abstract_image_tensor.py | 4 ++-- docarray/typing/tensor/tensorflow_tensor.py | 1 - docarray/typing/tensor/video/video_tensor_mixin.py | 2 +- docarray/typing/url/url_3d/url_3d.py | 2 +- docarray/utils/_internal/compress.py | 6 +++--- 12 files changed, 14 insertions(+), 15 deletions(-) diff --git a/docarray/array/array/io.py b/docarray/array/array/io.py index d6160cf9931..2b8d6a72ff3 100644 --- a/docarray/array/array/io.py +++ b/docarray/array/array/io.py @@ -504,7 +504,7 @@ def to_pandas(self) -> 'pd.DataFrame': if TYPE_CHECKING: import pandas as pd else: - pd = import_library('pandas') + pd = import_library('pandas', raise_error=True) fields = self.document_type._get_access_paths() df = pd.DataFrame(columns=fields) diff --git a/docarray/documents/mesh/vertices_and_faces.py b/docarray/documents/mesh/vertices_and_faces.py index 294bb28e61f..c64d5b7982a 100644 --- a/docarray/documents/mesh/vertices_and_faces.py +++ b/docarray/documents/mesh/vertices_and_faces.py @@ -34,7 +34,7 @@ def display(self) -> None: if TYPE_CHECKING: import trimesh else: - trimesh = import_library('trimesh') + trimesh = import_library('trimesh', raise_error=True) from IPython.display import display diff --git a/docarray/documents/point_cloud/points_and_colors.py b/docarray/documents/point_cloud/points_and_colors.py index 288c8caba3e..a54d79e838b 100644 --- a/docarray/documents/point_cloud/points_and_colors.py +++ b/docarray/documents/point_cloud/points_and_colors.py @@ -52,7 +52,7 @@ def display(self) -> None: if TYPE_CHECKING: import trimesh else: - trimesh = import_library('trimesh') + trimesh = import_library('trimesh', raise_error=True) from IPython.display import display colors = ( diff --git a/docarray/proto/__init__.py b/docarray/proto/__init__.py index 391b0027277..bca72030b79 100644 --- a/docarray/proto/__init__.py +++ b/docarray/proto/__init__.py @@ -5,7 +5,7 @@ if TYPE_CHECKING: from google.protobuf import __version__ as __pb__version__ else: - protobuf = import_library('google.protobuf') + protobuf = import_library('google.protobuf', raise_error=True) __pb__version__ = protobuf.__version__ diff --git a/docarray/typing/bytes/audio_bytes.py b/docarray/typing/bytes/audio_bytes.py index 01431093105..aae0a628d35 100644 --- a/docarray/typing/bytes/audio_bytes.py +++ b/docarray/typing/bytes/audio_bytes.py @@ -78,7 +78,7 @@ class MyAudio(Document): if TYPE_CHECKING: import pydub else: - pydub = import_library('pydub') + pydub = import_library('pydub', raise_error=True) segment = pydub.AudioSegment.from_file(io.BytesIO(self)) diff --git a/docarray/typing/bytes/image_bytes.py b/docarray/typing/bytes/image_bytes.py index cd75b02a6b1..2ab187d12f5 100644 --- a/docarray/typing/bytes/image_bytes.py +++ b/docarray/typing/bytes/image_bytes.py @@ -91,7 +91,7 @@ class MyDoc(BaseDoc): if TYPE_CHECKING: from PIL import Image as PILImage else: - PILImage = import_library('PIL').Image + PILImage = import_library('PIL', raise_error=True).Image raw_img = PILImage.open(BytesIO(self)) if width or height: diff --git a/docarray/typing/tensor/audio/abstract_audio_tensor.py b/docarray/typing/tensor/audio/abstract_audio_tensor.py index 40ecadacf0e..05a455e2c6c 100644 --- a/docarray/typing/tensor/audio/abstract_audio_tensor.py +++ b/docarray/typing/tensor/audio/abstract_audio_tensor.py @@ -40,7 +40,7 @@ def save( if TYPE_CHECKING: import pydub else: - pydub = import_library('pydub') + pydub = import_library('pydub', raise_error=True) comp_backend = self.get_comp_backend() channels = 2 if comp_backend.n_dim(array=self) > 1 else 1 # type: ignore diff --git a/docarray/typing/tensor/image/abstract_image_tensor.py b/docarray/typing/tensor/image/abstract_image_tensor.py index 46bb61ed803..fd262e4cdc5 100644 --- a/docarray/typing/tensor/image/abstract_image_tensor.py +++ b/docarray/typing/tensor/image/abstract_image_tensor.py @@ -18,7 +18,7 @@ def to_bytes(self, format: str = 'PNG') -> bytes: if TYPE_CHECKING: from PIL import Image as PILImage else: - PILImage = import_library('PIL').Image + PILImage = import_library('PIL', raise_error=True).Image if format == 'jpg': format = 'jpeg' # unify it to ISO standard @@ -42,7 +42,7 @@ def display(self) -> None: if TYPE_CHECKING: from PIL import Image as PILImage else: - PILImage = import_library('PIL').Image + PILImage = import_library('PIL', raise_error=True).Image np_array = self.get_comp_backend().to_numpy(self) img = PILImage.fromarray(np_array) diff --git a/docarray/typing/tensor/tensorflow_tensor.py b/docarray/typing/tensor/tensorflow_tensor.py index 6d60ea3d629..2329a8c689a 100644 --- a/docarray/typing/tensor/tensorflow_tensor.py +++ b/docarray/typing/tensor/tensorflow_tensor.py @@ -18,7 +18,6 @@ tf = import_library('tensorflow', raise_error=True) - T = TypeVar('T', bound='TensorFlowTensor') ShapeT = TypeVar('ShapeT') diff --git a/docarray/typing/tensor/video/video_tensor_mixin.py b/docarray/typing/tensor/video/video_tensor_mixin.py index b204a22ca63..7cdc1b82ff6 100644 --- a/docarray/typing/tensor/video/video_tensor_mixin.py +++ b/docarray/typing/tensor/video/video_tensor_mixin.py @@ -78,7 +78,7 @@ class MyDoc(BaseDoc): if TYPE_CHECKING: import av else: - av = import_library('av') + av = import_library('av', raise_error=True) np_tensor = self.get_comp_backend().to_numpy(array=self) video_tensor = np_tensor.astype('uint8') diff --git a/docarray/typing/url/url_3d/url_3d.py b/docarray/typing/url/url_3d/url_3d.py index a2210d8bd86..b4349251131 100644 --- a/docarray/typing/url/url_3d/url_3d.py +++ b/docarray/typing/url/url_3d/url_3d.py @@ -59,7 +59,7 @@ def _load_trimesh_instance( if TYPE_CHECKING: import trimesh else: - trimesh = import_library('trimesh') + trimesh = import_library('trimesh', raise_error=True) if not trimesh_args: trimesh_args = {} diff --git a/docarray/utils/_internal/compress.py b/docarray/utils/_internal/compress.py index e71136bb3b4..ab37ad1a694 100644 --- a/docarray/utils/_internal/compress.py +++ b/docarray/utils/_internal/compress.py @@ -8,7 +8,7 @@ def _compress_bytes(data: bytes, algorithm: Optional[str] = None) -> bytes: if TYPE_CHECKING: import lz4.frame as lz4_frame # type: ignore else: - lz4_frame = import_library('lz4').frame + lz4_frame = import_library('lz4', raise_error=True).frame data = lz4_frame.compress(data) elif algorithm == 'bz2': @@ -36,7 +36,7 @@ def _decompress_bytes(data: bytes, algorithm: Optional[str] = None) -> bytes: if TYPE_CHECKING: import lz4.frame as lz4_frame # type: ignore else: - lz4_frame = import_library('lz4').frame + lz4_frame = import_library('lz4', raise_error=True).frame data = lz4_frame.decompress(data) elif algorithm == 'bz2': @@ -64,7 +64,7 @@ def _get_compress_ctx(algorithm: Optional[str] = None) -> Optional[Callable]: if TYPE_CHECKING: import lz4.frame as lz4_frame # type: ignore else: - lz4_frame = import_library('lz4').frame + lz4_frame = import_library('lz4', raise_error=True).frame def _fun(x: IO[bytes]): return lz4_frame.LZ4FrameFile(x, 'wb') From be0727887603912cf43b6d6242e8caa2ae06e825 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 10:54:21 +0200 Subject: [PATCH 32/66] fix: add trimesh easy extra Signed-off-by: anna-charlotte --- docarray/documents/mesh/vertices_and_faces.py | 1 - .../point_cloud/points_and_colors.py | 1 - docarray/typing/url/url_3d/mesh_url.py | 1 - docarray/typing/url/url_3d/point_cloud_url.py | 1 - poetry.lock | 576 +++++++++++++++++- pyproject.toml | 2 +- 6 files changed, 565 insertions(+), 17 deletions(-) diff --git a/docarray/documents/mesh/vertices_and_faces.py b/docarray/documents/mesh/vertices_and_faces.py index c64d5b7982a..19697b1aa98 100644 --- a/docarray/documents/mesh/vertices_and_faces.py +++ b/docarray/documents/mesh/vertices_and_faces.py @@ -29,7 +29,6 @@ def validate( def display(self) -> None: """ Plot mesh consisting of vertices and faces. - To use this you need to install trimesh[easy]: `pip install 'trimesh[easy]'`. """ if TYPE_CHECKING: import trimesh diff --git a/docarray/documents/point_cloud/points_and_colors.py b/docarray/documents/point_cloud/points_and_colors.py index a54d79e838b..825744bda4f 100644 --- a/docarray/documents/point_cloud/points_and_colors.py +++ b/docarray/documents/point_cloud/points_and_colors.py @@ -47,7 +47,6 @@ def validate( def display(self) -> None: """ Plot point cloud consisting of points in 3D space and optionally colors. - To use this you need to install trimesh[easy]: `pip install 'trimesh[easy]'`. """ if TYPE_CHECKING: import trimesh diff --git a/docarray/typing/url/url_3d/mesh_url.py b/docarray/typing/url/url_3d/mesh_url.py index e820ba8a4d5..ce66cd3ad0b 100644 --- a/docarray/typing/url/url_3d/mesh_url.py +++ b/docarray/typing/url/url_3d/mesh_url.py @@ -71,7 +71,6 @@ def display(self) -> None: """ Plot mesh from url. This loads the Trimesh instance of the 3D mesh, and then displays it. - To use this you need to install trimesh[easy]: `pip install 'trimesh[easy]'`. """ from IPython.display import display diff --git a/docarray/typing/url/url_3d/point_cloud_url.py b/docarray/typing/url/url_3d/point_cloud_url.py index 502a29c740d..798c6a05aec 100644 --- a/docarray/typing/url/url_3d/point_cloud_url.py +++ b/docarray/typing/url/url_3d/point_cloud_url.py @@ -89,7 +89,6 @@ def display( ) -> None: """ Plot point cloud from url. - To use this you need to install trimesh[easy]: `pip install 'trimesh[easy]'`. First, it loads the point cloud into a :class:`PointsAndColors` object, and then calls display on it. The following is therefore equivalent: diff --git a/poetry.lock b/poetry.lock index 8c38a78ce56..f8e9abbb543 100644 --- a/poetry.lock +++ b/poetry.lock @@ -610,6 +610,18 @@ files = [ {file = "cfgv-3.3.1.tar.gz", hash = "sha256:f5a830efb9ce7a445376bb66ec94c638a9787422f96264c98edc6bdeed8ab736"}, ] +[[package]] +name = "chardet" +version = "5.1.0" +description = "Universal encoding detector for Python 3" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "chardet-5.1.0-py3-none-any.whl", hash = "sha256:362777fb014af596ad31334fde1e8c327dfdb076e1960d1694662d46a6917ab9"}, + {file = "chardet-5.1.0.tar.gz", hash = "sha256:0d62712b956bc154f85fb0a266e2a3c5913c2967e00348701b32411d6def31e5"}, +] + [[package]] name = "charset-normalizer" version = "2.0.12" @@ -645,7 +657,7 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "dev" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -653,6 +665,24 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "colorlog" +version = "6.7.0" +description = "Add colours to the output of Python's logging module." +category = "main" +optional = true +python-versions = ">=3.6" +files = [ + {file = "colorlog-6.7.0-py2.py3-none-any.whl", hash = "sha256:0d33ca236784a1ba3ff9c532d4964126d8a2c44f1f0cb1d2b0728196f512f662"}, + {file = "colorlog-6.7.0.tar.gz", hash = "sha256:bd94bd21c1e13fac7bd3153f4bc3a7dc0eb0974b8bc2fdf1a989e474f6e582e5"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +development = ["black", "flake8", "mypy", "pytest", "types-colorama"] + [[package]] name = "commonmark" version = "0.9.1" @@ -1101,7 +1131,7 @@ testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packag name = "importlib-resources" version = "5.10.0" description = "Read resources from Python packages" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1318,7 +1348,7 @@ dev = ["hypothesis"] name = "jsonschema" version = "4.17.0" description = "An implementation of JSON Schema validation for Python" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1482,6 +1512,99 @@ docs = ["autodoc-traits", "docutils (<0.19)", "jinja2 (<3.1.0)", "mistune (<1)", openapi = ["openapi-core (>=0.14.2)", "ruamel-yaml"] test = ["codecov", "ipykernel", "jupyter-server[test]", "openapi-core (>=0.14.2,<0.15.0)", "openapi-spec-validator (<0.5)", "pytest (>=7.0)", "pytest-console-scripts", "pytest-cov", "requests-mock", "ruamel-yaml", "strict-rfc3339"] +[[package]] +name = "lxml" +version = "4.9.2" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +category = "main" +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" +files = [ + {file = "lxml-4.9.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:76cf573e5a365e790396a5cc2b909812633409306c6531a6877c59061e42c4f2"}, + {file = "lxml-4.9.2-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b1f42b6921d0e81b1bcb5e395bc091a70f41c4d4e55ba99c6da2b31626c44892"}, + {file = "lxml-4.9.2-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9f102706d0ca011de571de32c3247c6476b55bb6bc65a20f682f000b07a4852a"}, + {file = "lxml-4.9.2-cp27-cp27m-win32.whl", hash = "sha256:8d0b4612b66ff5d62d03bcaa043bb018f74dfea51184e53f067e6fdcba4bd8de"}, + {file = "lxml-4.9.2-cp27-cp27m-win_amd64.whl", hash = "sha256:4c8f293f14abc8fd3e8e01c5bd86e6ed0b6ef71936ded5bf10fe7a5efefbaca3"}, + {file = "lxml-4.9.2-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2899456259589aa38bfb018c364d6ae7b53c5c22d8e27d0ec7609c2a1ff78b50"}, + {file = "lxml-4.9.2-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6749649eecd6a9871cae297bffa4ee76f90b4504a2a2ab528d9ebe912b101975"}, + {file = "lxml-4.9.2-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:a08cff61517ee26cb56f1e949cca38caabe9ea9fbb4b1e10a805dc39844b7d5c"}, + {file = "lxml-4.9.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:85cabf64adec449132e55616e7ca3e1000ab449d1d0f9d7f83146ed5bdcb6d8a"}, + {file = "lxml-4.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:8340225bd5e7a701c0fa98284c849c9b9fc9238abf53a0ebd90900f25d39a4e4"}, + {file = "lxml-4.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:1ab8f1f932e8f82355e75dda5413a57612c6ea448069d4fb2e217e9a4bed13d4"}, + {file = "lxml-4.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:699a9af7dffaf67deeae27b2112aa06b41c370d5e7633e0ee0aea2e0b6c211f7"}, + {file = "lxml-4.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b9cc34af337a97d470040f99ba4282f6e6bac88407d021688a5d585e44a23184"}, + {file = "lxml-4.9.2-cp310-cp310-win32.whl", hash = "sha256:d02a5399126a53492415d4906ab0ad0375a5456cc05c3fc0fc4ca11771745cda"}, + {file = "lxml-4.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:a38486985ca49cfa574a507e7a2215c0c780fd1778bb6290c21193b7211702ab"}, + {file = "lxml-4.9.2-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:c83203addf554215463b59f6399835201999b5e48019dc17f182ed5ad87205c9"}, + {file = "lxml-4.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:2a87fa548561d2f4643c99cd13131acb607ddabb70682dcf1dff5f71f781a4bf"}, + {file = "lxml-4.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:d6b430a9938a5a5d85fc107d852262ddcd48602c120e3dbb02137c83d212b380"}, + {file = "lxml-4.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3efea981d956a6f7173b4659849f55081867cf897e719f57383698af6f618a92"}, + {file = "lxml-4.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:df0623dcf9668ad0445e0558a21211d4e9a149ea8f5666917c8eeec515f0a6d1"}, + {file = "lxml-4.9.2-cp311-cp311-win32.whl", hash = "sha256:da248f93f0418a9e9d94b0080d7ebc407a9a5e6d0b57bb30db9b5cc28de1ad33"}, + {file = "lxml-4.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:3818b8e2c4b5148567e1b09ce739006acfaa44ce3156f8cbbc11062994b8e8dd"}, + {file = "lxml-4.9.2-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ca989b91cf3a3ba28930a9fc1e9aeafc2a395448641df1f387a2d394638943b0"}, + {file = "lxml-4.9.2-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:822068f85e12a6e292803e112ab876bc03ed1f03dddb80154c395f891ca6b31e"}, + {file = "lxml-4.9.2-cp35-cp35m-win32.whl", hash = "sha256:be7292c55101e22f2a3d4d8913944cbea71eea90792bf914add27454a13905df"}, + {file = "lxml-4.9.2-cp35-cp35m-win_amd64.whl", hash = "sha256:998c7c41910666d2976928c38ea96a70d1aa43be6fe502f21a651e17483a43c5"}, + {file = "lxml-4.9.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:b26a29f0b7fc6f0897f043ca366142d2b609dc60756ee6e4e90b5f762c6adc53"}, + {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:ab323679b8b3030000f2be63e22cdeea5b47ee0abd2d6a1dc0c8103ddaa56cd7"}, + {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:689bb688a1db722485e4610a503e3e9210dcc20c520b45ac8f7533c837be76fe"}, + {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:f49e52d174375a7def9915c9f06ec4e569d235ad428f70751765f48d5926678c"}, + {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:36c3c175d34652a35475a73762b545f4527aec044910a651d2bf50de9c3352b1"}, + {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a35f8b7fa99f90dd2f5dc5a9fa12332642f087a7641289ca6c40d6e1a2637d8e"}, + {file = "lxml-4.9.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:58bfa3aa19ca4c0f28c5dde0ff56c520fbac6f0daf4fac66ed4c8d2fb7f22e74"}, + {file = "lxml-4.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc718cd47b765e790eecb74d044cc8d37d58562f6c314ee9484df26276d36a38"}, + {file = "lxml-4.9.2-cp36-cp36m-win32.whl", hash = "sha256:d5bf6545cd27aaa8a13033ce56354ed9e25ab0e4ac3b5392b763d8d04b08e0c5"}, + {file = "lxml-4.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:3ab9fa9d6dc2a7f29d7affdf3edebf6ece6fb28a6d80b14c3b2fb9d39b9322c3"}, + {file = "lxml-4.9.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:05ca3f6abf5cf78fe053da9b1166e062ade3fa5d4f92b4ed688127ea7d7b1d03"}, + {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:a5da296eb617d18e497bcf0a5c528f5d3b18dadb3619fbdadf4ed2356ef8d941"}, + {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:04876580c050a8c5341d706dd464ff04fd597095cc8c023252566a8826505726"}, + {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c9ec3eaf616d67db0764b3bb983962b4f385a1f08304fd30c7283954e6a7869b"}, + {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2a29ba94d065945944016b6b74e538bdb1751a1db6ffb80c9d3c2e40d6fa9894"}, + {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a82d05da00a58b8e4c0008edbc8a4b6ec5a4bc1e2ee0fb6ed157cf634ed7fa45"}, + {file = "lxml-4.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:223f4232855ade399bd409331e6ca70fb5578efef22cf4069a6090acc0f53c0e"}, + {file = "lxml-4.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d17bc7c2ccf49c478c5bdd447594e82692c74222698cfc9b5daae7ae7e90743b"}, + {file = "lxml-4.9.2-cp37-cp37m-win32.whl", hash = "sha256:b64d891da92e232c36976c80ed7ebb383e3f148489796d8d31a5b6a677825efe"}, + {file = "lxml-4.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:a0a336d6d3e8b234a3aae3c674873d8f0e720b76bc1d9416866c41cd9500ffb9"}, + {file = "lxml-4.9.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:da4dd7c9c50c059aba52b3524f84d7de956f7fef88f0bafcf4ad7dde94a064e8"}, + {file = "lxml-4.9.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:821b7f59b99551c69c85a6039c65b75f5683bdc63270fec660f75da67469ca24"}, + {file = "lxml-4.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:e5168986b90a8d1f2f9dc1b841467c74221bd752537b99761a93d2d981e04889"}, + {file = "lxml-4.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8e20cb5a47247e383cf4ff523205060991021233ebd6f924bca927fcf25cf86f"}, + {file = "lxml-4.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:13598ecfbd2e86ea7ae45ec28a2a54fb87ee9b9fdb0f6d343297d8e548392c03"}, + {file = "lxml-4.9.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:880bbbcbe2fca64e2f4d8e04db47bcdf504936fa2b33933efd945e1b429bea8c"}, + {file = "lxml-4.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7d2278d59425777cfcb19735018d897ca8303abe67cc735f9f97177ceff8027f"}, + {file = "lxml-4.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5344a43228767f53a9df6e5b253f8cdca7dfc7b7aeae52551958192f56d98457"}, + {file = "lxml-4.9.2-cp38-cp38-win32.whl", hash = "sha256:925073b2fe14ab9b87e73f9a5fde6ce6392da430f3004d8b72cc86f746f5163b"}, + {file = "lxml-4.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:9b22c5c66f67ae00c0199f6055705bc3eb3fcb08d03d2ec4059a2b1b25ed48d7"}, + {file = "lxml-4.9.2-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:5f50a1c177e2fa3ee0667a5ab79fdc6b23086bc8b589d90b93b4bd17eb0e64d1"}, + {file = "lxml-4.9.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:090c6543d3696cbe15b4ac6e175e576bcc3f1ccfbba970061b7300b0c15a2140"}, + {file = "lxml-4.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:63da2ccc0857c311d764e7d3d90f429c252e83b52d1f8f1d1fe55be26827d1f4"}, + {file = "lxml-4.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:5b4545b8a40478183ac06c073e81a5ce4cf01bf1734962577cf2bb569a5b3bbf"}, + {file = "lxml-4.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2e430cd2824f05f2d4f687701144556646bae8f249fd60aa1e4c768ba7018947"}, + {file = "lxml-4.9.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6804daeb7ef69e7b36f76caddb85cccd63d0c56dedb47555d2fc969e2af6a1a5"}, + {file = "lxml-4.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a6e441a86553c310258aca15d1c05903aaf4965b23f3bc2d55f200804e005ee5"}, + {file = "lxml-4.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ca34efc80a29351897e18888c71c6aca4a359247c87e0b1c7ada14f0ab0c0fb2"}, + {file = "lxml-4.9.2-cp39-cp39-win32.whl", hash = "sha256:6b418afe5df18233fc6b6093deb82a32895b6bb0b1155c2cdb05203f583053f1"}, + {file = "lxml-4.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1496ea22ca2c830cbcbd473de8f114a320da308438ae65abad6bab7867fe38f"}, + {file = "lxml-4.9.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:b264171e3143d842ded311b7dccd46ff9ef34247129ff5bf5066123c55c2431c"}, + {file = "lxml-4.9.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0dc313ef231edf866912e9d8f5a042ddab56c752619e92dfd3a2c277e6a7299a"}, + {file = "lxml-4.9.2-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:16efd54337136e8cd72fb9485c368d91d77a47ee2d42b057564aae201257d419"}, + {file = "lxml-4.9.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:0f2b1e0d79180f344ff9f321327b005ca043a50ece8713de61d1cb383fb8ac05"}, + {file = "lxml-4.9.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:7b770ed79542ed52c519119473898198761d78beb24b107acf3ad65deae61f1f"}, + {file = "lxml-4.9.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:efa29c2fe6b4fdd32e8ef81c1528506895eca86e1d8c4657fda04c9b3786ddf9"}, + {file = "lxml-4.9.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7e91ee82f4199af8c43d8158024cbdff3d931df350252288f0d4ce656df7f3b5"}, + {file = "lxml-4.9.2-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:b23e19989c355ca854276178a0463951a653309fb8e57ce674497f2d9f208746"}, + {file = "lxml-4.9.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:01d36c05f4afb8f7c20fd9ed5badca32a2029b93b1750f571ccc0b142531caf7"}, + {file = "lxml-4.9.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7b515674acfdcadb0eb5d00d8a709868173acece5cb0be3dd165950cbfdf5409"}, + {file = "lxml-4.9.2.tar.gz", hash = "sha256:2455cfaeb7ac70338b3257f41e21f0724f4b5b0c0e7702da67ee6c3640835b67"}, +] + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html5 = ["html5lib"] +htmlsoup = ["BeautifulSoup4"] +source = ["Cython (>=0.29.7)"] + [[package]] name = "lz4" version = "4.3.2" @@ -1532,6 +1655,81 @@ docs = ["sphinx (>=1.6.0)", "sphinx-bootstrap-theme"] flake8 = ["flake8"] tests = ["psutil", "pytest (!=3.3.0)", "pytest-cov"] +[[package]] +name = "mapbox-earcut" +version = "1.0.1" +description = "Python bindings for the mapbox earcut C++ polygon triangulation library." +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "mapbox_earcut-1.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:60f8299b724b5ad1f171c2666a12591845536b0e9318ddc9649f75805096686c"}, + {file = "mapbox_earcut-1.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4af0911ed9d1920c36c54b500ea69fbcc948f409c66f632c75b15fee04c7544e"}, + {file = "mapbox_earcut-1.0.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:584fd2f7de878f14b3268257ec3c55bac146f1adc1887a64f0ecbf91ee39489f"}, + {file = "mapbox_earcut-1.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20929541c1c9f5fefde45c6c33e8ed3138c7bdd1034ced998877913878f3457c"}, + {file = "mapbox_earcut-1.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:48e8d8ebadd4e4d0dfd87374d43ca3caf8c8e692f1b6897588594d12527d5020"}, + {file = "mapbox_earcut-1.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:352f92997fd39024919a258db29df1642dd98632807ca96e737242adf64b5e96"}, + {file = "mapbox_earcut-1.0.1-cp310-cp310-win32.whl", hash = "sha256:5cf359c5ae1a5dcdd6d9c150ec43a820a289c28596ae7c52de09075543cc19ae"}, + {file = "mapbox_earcut-1.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f5cd49d6e13b3627c6cd6d3a945285e1ce7e9b193f3ce5ca53f0b7b86acd41e"}, + {file = "mapbox_earcut-1.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1e02d61d01aa1239ffbe1b8384cdc224d7c67db604eb7bfc34dd39fb1dc515c2"}, + {file = "mapbox_earcut-1.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d170d0a79b4ef3c9591ec6727a0ab35bae9e267b389122365343d6f55f9027a0"}, + {file = "mapbox_earcut-1.0.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78945356229992d7aa6da750059f401f329651adc76c000505a0e9e4f93be5df"}, + {file = "mapbox_earcut-1.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66cf29a2434d3366889c69fc50e6d2f9f1abf3a8a4154c7e03ef8f180d3bea40"}, + {file = "mapbox_earcut-1.0.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:a73f6f422932b2758b03f78e92aa5c4d5b5f7ce6456483f5993f4677b0bbde23"}, + {file = "mapbox_earcut-1.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9af9369266bf0ca32f4d401152217c46c699392513f22639c6b1be32bde9c1cc"}, + {file = "mapbox_earcut-1.0.1-cp311-cp311-win32.whl", hash = "sha256:ff9a13be4364625697b0e0e04ba6a0f77300148b871bba0a85bfa67e972e85c4"}, + {file = "mapbox_earcut-1.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:5e736557539c74fa969e866889c2b0149fc12668f35e3ae33667d837ff2880d3"}, + {file = "mapbox_earcut-1.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:aa6111a18efacb79c081f3d3cdd7d25d0585bb0e9f28896b207ebe1d56efa40e"}, + {file = "mapbox_earcut-1.0.1-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f2911829d1e6e5e1282fbe2840fadf578f606580f02ed436346c2d51c92f810b"}, + {file = "mapbox_earcut-1.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01ff909a7b8405a923abedd701b53633c997cc2b5dc9d5b78462f51c25ec2c33"}, + {file = "mapbox_earcut-1.0.1-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:b7e73477b4ef3951ef5c32848126f047ac7fd2dd04dc033444a85261a346ed08"}, + {file = "mapbox_earcut-1.0.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:dae325af3553afa4d0ca0caa5afe57dc3d2e3a90a51dfbabc49a5ce1ea1009f7"}, + {file = "mapbox_earcut-1.0.1-cp36-cp36m-win32.whl", hash = "sha256:29f8f746a9c68f1509084b0c78227d4e142241a2e30aab6def872e53a46f7281"}, + {file = "mapbox_earcut-1.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:3c487b93b0e1059b404be4daea62c22cfc8054ffd88591377848c8e399d4abeb"}, + {file = "mapbox_earcut-1.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f85f8d95503dba4612a2dd5c076ed18845a46cea4ba38660e4929efccb5a594a"}, + {file = "mapbox_earcut-1.0.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8ade6c4822be1680c933bda32af0bb23a73e63e951db348ac1adef8de137239"}, + {file = "mapbox_earcut-1.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:04431a498a836c62aba5d807572daf3c8b064b25ab83e79994498455524ce517"}, + {file = "mapbox_earcut-1.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:714d33603c59d7306650615d7b05d51da273f1aa5b41c3b462207271a2283fa7"}, + {file = "mapbox_earcut-1.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:065faa6b4a7525faa48e46e692176cbcf9587ade7a1abdb2c96cb6477ab0004d"}, + {file = "mapbox_earcut-1.0.1-cp37-cp37m-win32.whl", hash = "sha256:9f433276f54e302aa0c3ef0f8edb7a4092cdd677aafc623fab2b81e1db9f2729"}, + {file = "mapbox_earcut-1.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:2ac93a18a19acffaa7dc42646534f3850b545d6ad31469f3b7157efc9da113db"}, + {file = "mapbox_earcut-1.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b23d0b41d5d7e72fa197e981c3e317f234336b4594bb64252837a0558c9c505d"}, + {file = "mapbox_earcut-1.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:57337d9cf95a97b926eab57845525501df61abb0334ed59502a6485cf9216f64"}, + {file = "mapbox_earcut-1.0.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5447f35b1dda5f89a6d5c95e9a1831f1c5aaf1eeac853f0b2f3df97ec81c2c75"}, + {file = "mapbox_earcut-1.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dffb3e2a0253e3e2e1b7638df80a029d4d80f38db42a7736f92a8e8d4d1a3209"}, + {file = "mapbox_earcut-1.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:202761e67b0974b1618e638b83a1bb24d0a421a0c773435833a368b9b4f0ee2b"}, + {file = "mapbox_earcut-1.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:9c37424997c46f45f16a8ec42fc892a011f9528257f207e2aae4bd14cfcd7c3d"}, + {file = "mapbox_earcut-1.0.1-cp38-cp38-win32.whl", hash = "sha256:ed5ec84c85a6e6cbfa294bdcbf567d3fa0abec9191acc8f362552946b8b7b398"}, + {file = "mapbox_earcut-1.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:da5eeb66f50b01e77340b00f29867fa89df4b9e28646f9a0b8f6b5c8827515fd"}, + {file = "mapbox_earcut-1.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eb4aa9a7d1c5f92458d73f460d1f063fbcb38c50ff1f0b7e3485b8dc0f1f6635"}, + {file = "mapbox_earcut-1.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f7f779084b11bd74be374be69054803ac36095a68d1a0da1d499a47d3c7f7ccc"}, + {file = "mapbox_earcut-1.0.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5190425932e82e22e3e35dfb892f5eb441aef155c45fa055da027c72c124b3d1"}, + {file = "mapbox_earcut-1.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d68b47dc4ab2aaa9ec163c18bc6133c74739990b5013d17e13bac2d1b5c9afea"}, + {file = "mapbox_earcut-1.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e1b737d4b7b1c52c3915b898714e036990149a422343ec1481ac66b35df17f24"}, + {file = "mapbox_earcut-1.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3b77f444324a3b0e91ba2b4b2d533a66503f8fb7103e4901d0064ec2413bff8c"}, + {file = "mapbox_earcut-1.0.1-cp39-cp39-win32.whl", hash = "sha256:db61cec2374ff063e314c40b3a868237d2af1b0d98f3ec1217bc0f881e7cc40a"}, + {file = "mapbox_earcut-1.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:86b8c3732fb93f4e8ed8b1cc8388b93a72d0e9755a00f324e780b15a00fe5bc0"}, + {file = "mapbox_earcut-1.0.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0b5ad819f3fd57fc8a18c7b61a244e63b2a24475195f57e826a066e007a7a877"}, + {file = "mapbox_earcut-1.0.1-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:732e5c86037692f6c635dc4e139520be8366cde0fd39dbe122480f657b2cca90"}, + {file = "mapbox_earcut-1.0.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6cf7c0d0d862addc99fe0b33150c8f5c06baafa320b6dd6f67d17309512d1e9a"}, + {file = "mapbox_earcut-1.0.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:8416071bd3af616afab4513347b064274899f73e0ffe309c2a1be66600736c98"}, + {file = "mapbox_earcut-1.0.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1310c3e208e0bfd6da090ae65226ee49adba4078fe1ed2d95197c3b97ad513b9"}, + {file = "mapbox_earcut-1.0.1-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b657a30f677de4005f497c79ab3bb2827ba01e2642cb58ac30242f7cff48e40b"}, + {file = "mapbox_earcut-1.0.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:34e3476d9af878887fd0d9cce759d6951fe0cc6c240e13afed1ff38fc23fc9d5"}, + {file = "mapbox_earcut-1.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e480ce4794b0c391f0b829362c78ec74b690104ef36866160a7e14232b2d3779"}, + {file = "mapbox_earcut-1.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:c21271dd89263d037af5caeac425e54a8fba727ea30d1b42e3ce94cc675df15a"}, + {file = "mapbox_earcut-1.0.1-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11c784ba52c981dcf709bcc8de99d75a214a476f7c16369d219ca4751c7f6f6f"}, + {file = "mapbox_earcut-1.0.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be37a75c94017a2efaffc8763475867d4860fc4cb3262b6839d635690403d28f"}, + {file = "mapbox_earcut-1.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ad50f947d44c8c1c0900c3e1869a4a550509450117b87b0368b06014f66590b"}, + {file = "mapbox_earcut-1.0.1.tar.gz", hash = "sha256:9f155e429a22e27387cfd7a6372c3a3865aafa609ad725e2c4465257f154a438"}, +] + +[package.dependencies] +numpy = "*" + +[package.extras] +test = ["pytest"] + [[package]] name = "markdown" version = "3.3.7" @@ -1794,6 +1992,24 @@ files = [ [package.extras] test = ["pytest (>=4.0.2)"] +[[package]] +name = "mpmath" +version = "1.3.0" +description = "Python library for arbitrary-precision floating-point arithmetic" +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c"}, + {file = "mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f"}, +] + +[package.extras] +develop = ["codecov", "pycodestyle", "pytest (>=4.6)", "pytest-cov", "wheel"] +docs = ["sphinx"] +gmpy = ["gmpy2 (>=2.1.0a4)"] +tests = ["pytest (>=4.6)"] + [[package]] name = "multidict" version = "6.0.4" @@ -2085,6 +2301,25 @@ files = [ {file = "nest_asyncio-1.5.6.tar.gz", hash = "sha256:d267cc1ff794403f7df692964d1d2a3fa9418ffea2a3f6859a439ff482fef290"}, ] +[[package]] +name = "networkx" +version = "2.6.3" +description = "Python package for creating and manipulating graphs and networks" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "networkx-2.6.3-py3-none-any.whl", hash = "sha256:80b6b89c77d1dfb64a4c7854981b60aeea6360ac02c6d4e4913319e0a313abef"}, + {file = "networkx-2.6.3.tar.gz", hash = "sha256:c0946ed31d71f1b732b5aaa6da5a0388a345019af232ce2f49c766e2d6795c51"}, +] + +[package.extras] +default = ["matplotlib (>=3.3)", "numpy (>=1.19)", "pandas (>=1.1)", "scipy (>=1.5,!=1.6.1)"] +developer = ["black (==21.5b1)", "pre-commit (>=2.12)"] +doc = ["nb2plots (>=0.6)", "numpydoc (>=1.1)", "pillow (>=8.2)", "pydata-sphinx-theme (>=0.6,<1.0)", "sphinx (>=4.0,<5.0)", "sphinx-gallery (>=0.9,<1.0)", "texext (>=0.6.6)"] +extra = ["lxml (>=4.5)", "pydot (>=1.4.1)", "pygraphviz (>=1.7)"] +test = ["codecov (>=2.1)", "pytest (>=6.2)", "pytest-cov (>=2.12)"] + [[package]] name = "nodeenv" version = "1.7.0" @@ -2510,7 +2745,7 @@ tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "pa name = "pkgutil-resolve-name" version = "1.3.10" description = "Resolve a name to an object." -category = "dev" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2690,6 +2925,24 @@ files = [ {file = "pyasn1-0.4.8.tar.gz", hash = "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba"}, ] +[[package]] +name = "pycollada" +version = "0.7.2" +description = "python library for reading and writing collada documents" +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "pycollada-0.7.2.tar.gz", hash = "sha256:70a2630ed499bdab718c0e61a3e6ae3698130d7e4654e89cdecde51bfdaea56f"}, +] + +[package.dependencies] +python-dateutil = ">=2.2" + +[package.extras] +prettyprint = ["lxml"] +validation = ["lxml"] + [[package]] name = "pycparser" version = "2.21" @@ -2817,7 +3070,7 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "pyrsistent" version = "0.19.2" description = "Persistent/Functional/Immutable data structures" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3304,6 +3557,64 @@ files = [ [package.dependencies] pyasn1 = ">=0.1.3" +[[package]] +name = "rtree" +version = "1.0.1" +description = "R-Tree spatial index for Python GIS" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "Rtree-1.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9855b8f11cdad99c56eb361b7b632a4fbd3d8cbe3f2081426b445f0cfb7fdca9"}, + {file = "Rtree-1.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:18ce7e4d04b85c48f2d364835620b3b20e38e199639746e7b12f07a2303e18ff"}, + {file = "Rtree-1.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:784efa6b7be9e99b33613ae8495931032689441eabb6120c9b3eb91188c33794"}, + {file = "Rtree-1.0.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:157207191aebdacbbdbb369e698cfbfebce53bc97114e96c8af5bed3126475f1"}, + {file = "Rtree-1.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5fb3671a8d440c24b1dd29ec621d4345ced7185e26f02abe98e85a6629fcb50"}, + {file = "Rtree-1.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:11d16f51cf9205cd6995af36e24efe8f184270f667fb49bb69b09fc46b97e7d4"}, + {file = "Rtree-1.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6db6a0a93e41594ffc14b053f386dd414ab5a82535bbd9aedafa6ac8dc0650d8"}, + {file = "Rtree-1.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c6e29e5eb3083ad12ac5c1ce6e37465ea3428d894d3466cc9c9e2ee4bf768e53"}, + {file = "Rtree-1.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:656b148589c0b5bab4a7db4d033634329f42a5feaac10ca40aceeca109d83c1f"}, + {file = "Rtree-1.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7b2c15f9373ba314c83a8df5cb6d99b4e3af23c376c6b1317add995432dd0970"}, + {file = "Rtree-1.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93c5e0bf31e76b4f92a6eec3d2891e938408774c75a8ed6ac3d2c8db04a2be33"}, + {file = "Rtree-1.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6792de0e3c2fd3ad7e069445027603bec7a47000432f49c80246886311f4f152"}, + {file = "Rtree-1.0.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:004e131b570dc360a49e7f3b60e7bc6517943a54df056587964d1cb903889e7e"}, + {file = "Rtree-1.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:becd711fe97c2e09b1b7969e83080a3c8012bce2d30f6db879aade255fcba5c1"}, + {file = "Rtree-1.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:015df09e1bc55ddf7c88799bf1515d058cd0ee78eacf4cd443a32876d3b3a863"}, + {file = "Rtree-1.0.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c2973b76f61669a85e160b4ad09879c4089fc0e3f20fd99adf161ca298fe8374"}, + {file = "Rtree-1.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e4335e131a58952635560a003458011d97f9ea6f3c010dc24906050b42ee2c03"}, + {file = "Rtree-1.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:e7ca5d743f6a1dc62653dfac8ee7ce2e1ba91be7cf97916a7f60b7cbe48fb48d"}, + {file = "Rtree-1.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:2ee7165e9872a026ccb868c021711eba39cedf7d1820763c9de52d5324691a92"}, + {file = "Rtree-1.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8de99f28af0f1783eefb80918959903b4b18112f6a12b48f296ecb162804e69d"}, + {file = "Rtree-1.0.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a94e2f4bf74bd202ea8b67ea3d7c71e763ad41f79be1d6b72aa2c8d5a8e92c4"}, + {file = "Rtree-1.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5120da3a1b96f3a7a17dd6af0afdd4e6f3cc9baa87e9ee0a272882f01f980bb"}, + {file = "Rtree-1.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7e3d5f0e7b28250afbb290ab88b49aa0f121c9714d0da2080581783690347507"}, + {file = "Rtree-1.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:296203e933b6ec0dd07f6a7456c4f1492def95b6993f20cc61c92b0fee0aecc5"}, + {file = "Rtree-1.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:77908cd7acdd519a731979ebf5baff8afd102109c2f52864c1e6ee75d3ea2d87"}, + {file = "Rtree-1.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:1a213e5d385278ca7668bc5b27083f8d6e39996a9bd59b6528f3a30009dae4ed"}, + {file = "Rtree-1.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cfa8cffec5cb9fed494c4bb335ebdb69b3c26178b0b685f67f79296c6b3d800c"}, + {file = "Rtree-1.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b31fd22d214160859d038da7cb2aaa27acb71efc24a7bcc75c84b5e502721549"}, + {file = "Rtree-1.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d68a81ad419d5c2ea5fecc677e6c178666c057e2c7b24100a6c48392196f1e9"}, + {file = "Rtree-1.0.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62f38020af47b765adc6b0bc7c4e810c6c3d1eab44ba339b592ff25a4c0dc0a7"}, + {file = "Rtree-1.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50b658a6707f215a0056d52e9f83a97148c0af62dea07cf29b3789a2c429e78a"}, + {file = "Rtree-1.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3573cbb0de872f54d0a0c29596a84e8ac3939c47ca3bece4a82e92775730a0d0"}, + {file = "Rtree-1.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d5abe5a19d943a88bea14901970e4c53e4579fc2662404cdea6163bf4c04d49a"}, + {file = "Rtree-1.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1e894112cef4de6c518bdea0b43eada65f12888c3645cc437c3a677aa023039f"}, + {file = "Rtree-1.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:582854252b8fd5c8472478af060635434931fb55edd269bac128cbf2eef43620"}, + {file = "Rtree-1.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b54057e8a8ad92c1d8e9eaa5cf32aad70dde454abbf9b638e9d6024520a52c02"}, + {file = "Rtree-1.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:698de8ce6c62e159d93b35bacf64bcf3619077b5367bc88cd2cff5e0bc36169b"}, + {file = "Rtree-1.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:273ee61783de3a1664e5f868feebf5eea4629447137751bfa4087b0f82093082"}, + {file = "Rtree-1.0.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:16900ee02cf5c198a42b03635268a80f606aa102f3f7618b89f75023d406da1c"}, + {file = "Rtree-1.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ce4a6fdb63254a4c1efebe7a4f7a59b1c333c703bde4ae715d9ad88c833e10b"}, + {file = "Rtree-1.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5b20f69e040a05503b22297af223f336fe7047909b57e4b207b98292f33a229f"}, + {file = "Rtree-1.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:57128293dd625cb1f07726f32208097953e8854d70ab1fc55d6858733618b9ed"}, + {file = "Rtree-1.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e898d7409ab645c25e06d4e058f99271182601d70b2887aba3351bf08e09a0c6"}, + {file = "Rtree-1.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:ad9912faeddb1ddcec5e26b33089166d58a107af6862d8b7f1bb2b7c0002ab39"}, + {file = "Rtree-1.0.1.tar.gz", hash = "sha256:222121699c303a64065d849bf7038b1ecabc37b65c7fa340bedb38ef0e805429"}, +] + +[package.dependencies] +typing-extensions = {version = ">=3.7", markers = "python_version < \"3.8\""} + [[package]] name = "ruff" version = "0.0.243" @@ -3348,6 +3659,38 @@ botocore = ">=1.12.36,<2.0a.0" [package.extras] crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] +[[package]] +name = "scipy" +version = "1.6.1" +description = "SciPy: Scientific Library for Python" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "scipy-1.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a15a1f3fc0abff33e792d6049161b7795909b40b97c6cc2934ed54384017ab76"}, + {file = "scipy-1.6.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:e79570979ccdc3d165456dd62041d9556fb9733b86b4b6d818af7a0afc15f092"}, + {file = "scipy-1.6.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:a423533c55fec61456dedee7b6ee7dce0bb6bfa395424ea374d25afa262be261"}, + {file = "scipy-1.6.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:33d6b7df40d197bdd3049d64e8e680227151673465e5d85723b3b8f6b15a6ced"}, + {file = "scipy-1.6.1-cp37-cp37m-win32.whl", hash = "sha256:6725e3fbb47da428794f243864f2297462e9ee448297c93ed1dcbc44335feb78"}, + {file = "scipy-1.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:5fa9c6530b1661f1370bcd332a1e62ca7881785cc0f80c0d559b636567fab63c"}, + {file = "scipy-1.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:bd50daf727f7c195e26f27467c85ce653d41df4358a25b32434a50d8870fc519"}, + {file = "scipy-1.6.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:f46dd15335e8a320b0fb4685f58b7471702234cba8bb3442b69a3e1dc329c345"}, + {file = "scipy-1.6.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:0e5b0ccf63155d90da576edd2768b66fb276446c371b73841e3503be1d63fb5d"}, + {file = "scipy-1.6.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:2481efbb3740977e3c831edfd0bd9867be26387cacf24eb5e366a6a374d3d00d"}, + {file = "scipy-1.6.1-cp38-cp38-win32.whl", hash = "sha256:68cb4c424112cd4be886b4d979c5497fba190714085f46b8ae67a5e4416c32b4"}, + {file = "scipy-1.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:5f331eeed0297232d2e6eea51b54e8278ed8bb10b099f69c44e2558c090d06bf"}, + {file = "scipy-1.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0c8a51d33556bf70367452d4d601d1742c0e806cd0194785914daf19775f0e67"}, + {file = "scipy-1.6.1-cp39-cp39-manylinux1_i686.whl", hash = "sha256:83bf7c16245c15bc58ee76c5418e46ea1811edcc2e2b03041b804e46084ab627"}, + {file = "scipy-1.6.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:794e768cc5f779736593046c9714e0f3a5940bc6dcc1dba885ad64cbfb28e9f0"}, + {file = "scipy-1.6.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:5da5471aed911fe7e52b86bf9ea32fb55ae93e2f0fac66c32e58897cfb02fa07"}, + {file = "scipy-1.6.1-cp39-cp39-win32.whl", hash = "sha256:8e403a337749ed40af60e537cc4d4c03febddcc56cd26e774c9b1b600a70d3e4"}, + {file = "scipy-1.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:a5193a098ae9f29af283dcf0041f762601faf2e595c0db1da929875b7570353f"}, + {file = "scipy-1.6.1.tar.gz", hash = "sha256:c4fceb864890b6168e79b0e714c585dbe2fd4222768ee90bc1aa0f8218691b11"}, +] + +[package.dependencies] +numpy = ">=1.16.5" + [[package]] name = "send2trash" version = "1.8.0" @@ -3382,6 +3725,61 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-g testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +[[package]] +name = "shapely" +version = "2.0.1" +description = "Manipulation and analysis of geometric objects" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "shapely-2.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b06d031bc64149e340448fea25eee01360a58936c89985cf584134171e05863f"}, + {file = "shapely-2.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9a6ac34c16f4d5d3c174c76c9d7614ec8fe735f8f82b6cc97a46b54f386a86bf"}, + {file = "shapely-2.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:865bc3d7cc0ea63189d11a0b1120d1307ed7a64720a8bfa5be2fde5fc6d0d33f"}, + {file = "shapely-2.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45b4833235b90bc87ee26c6537438fa77559d994d2d3be5190dd2e54d31b2820"}, + {file = "shapely-2.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce88ec79df55430e37178a191ad8df45cae90b0f6972d46d867bf6ebbb58cc4d"}, + {file = "shapely-2.0.1-cp310-cp310-win32.whl", hash = "sha256:01224899ff692a62929ef1a3f5fe389043e262698a708ab7569f43a99a48ae82"}, + {file = "shapely-2.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:da71de5bf552d83dcc21b78cc0020e86f8d0feea43e202110973987ffa781c21"}, + {file = "shapely-2.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:502e0a607f1dcc6dee0125aeee886379be5242c854500ea5fd2e7ac076b9ce6d"}, + {file = "shapely-2.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7d3bbeefd8a6a1a1017265d2d36f8ff2d79d0162d8c141aa0d37a87063525656"}, + {file = "shapely-2.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f470a130d6ddb05b810fc1776d918659407f8d025b7f56d2742a596b6dffa6c7"}, + {file = "shapely-2.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4641325e065fd3e07d55677849c9ddfd0cf3ee98f96475126942e746d55b17c8"}, + {file = "shapely-2.0.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:90cfa4144ff189a3c3de62e2f3669283c98fb760cfa2e82ff70df40f11cadb39"}, + {file = "shapely-2.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70a18fc7d6418e5aea76ac55dce33f98e75bd413c6eb39cfed6a1ba36469d7d4"}, + {file = "shapely-2.0.1-cp311-cp311-win32.whl", hash = "sha256:09d6c7763b1bee0d0a2b84bb32a4c25c6359ad1ac582a62d8b211e89de986154"}, + {file = "shapely-2.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:d8f55f355be7821dade839df785a49dc9f16d1af363134d07eb11e9207e0b189"}, + {file = "shapely-2.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:83a8ec0ee0192b6e3feee9f6a499d1377e9c295af74d7f81ecba5a42a6b195b7"}, + {file = "shapely-2.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a529218e72a3dbdc83676198e610485fdfa31178f4be5b519a8ae12ea688db14"}, + {file = "shapely-2.0.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:91575d97fd67391b85686573d758896ed2fc7476321c9d2e2b0c398b628b961c"}, + {file = "shapely-2.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8b0d834b11be97d5ab2b4dceada20ae8e07bcccbc0f55d71df6729965f406ad"}, + {file = "shapely-2.0.1-cp37-cp37m-win32.whl", hash = "sha256:b4f0711cc83734c6fad94fc8d4ec30f3d52c1787b17d9dca261dc841d4731c64"}, + {file = "shapely-2.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:05c51a29336e604c084fb43ae5dbbfa2c0ef9bd6fedeae0a0d02c7b57a56ba46"}, + {file = "shapely-2.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b519cf3726ddb6c67f6a951d1bb1d29691111eaa67ea19ddca4d454fbe35949c"}, + {file = "shapely-2.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:193a398d81c97a62fc3634a1a33798a58fd1dcf4aead254d080b273efbb7e3ff"}, + {file = "shapely-2.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e55698e0ed95a70fe9ff9a23c763acfe0bf335b02df12142f74e4543095e9a9b"}, + {file = "shapely-2.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f32a748703e7bf6e92dfa3d2936b2fbfe76f8ce5f756e24f49ef72d17d26ad02"}, + {file = "shapely-2.0.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a34a23d6266ca162499e4a22b79159dc0052f4973d16f16f990baa4d29e58b6"}, + {file = "shapely-2.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d173d24e85e51510e658fb108513d5bc11e3fd2820db6b1bd0522266ddd11f51"}, + {file = "shapely-2.0.1-cp38-cp38-win32.whl", hash = "sha256:3cb256ae0c01b17f7bc68ee2ffdd45aebf42af8992484ea55c29a6151abe4386"}, + {file = "shapely-2.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:c7eed1fb3008a8a4a56425334b7eb82651a51f9e9a9c2f72844a2fb394f38a6c"}, + {file = "shapely-2.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ac1dfc397475d1de485e76de0c3c91cc9d79bd39012a84bb0f5e8a199fc17bef"}, + {file = "shapely-2.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:33403b8896e1d98aaa3a52110d828b18985d740cc9f34f198922018b1e0f8afe"}, + {file = "shapely-2.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2569a4b91caeef54dd5ae9091ae6f63526d8ca0b376b5bb9fd1a3195d047d7d4"}, + {file = "shapely-2.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a70a614791ff65f5e283feed747e1cc3d9e6c6ba91556e640636bbb0a1e32a71"}, + {file = "shapely-2.0.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c43755d2c46b75a7b74ac6226d2cc9fa2a76c3263c5ae70c195c6fb4e7b08e79"}, + {file = "shapely-2.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad81f292fffbd568ae71828e6c387da7eb5384a79db9b4fde14dd9fdeffca9a"}, + {file = "shapely-2.0.1-cp39-cp39-win32.whl", hash = "sha256:b50c401b64883e61556a90b89948297f1714dbac29243d17ed9284a47e6dd731"}, + {file = "shapely-2.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:bca57b683e3d94d0919e2f31e4d70fdfbb7059650ef1b431d9f4e045690edcd5"}, + {file = "shapely-2.0.1.tar.gz", hash = "sha256:66a6b1a3e72ece97fc85536a281476f9b7794de2e646ca8a4517e2e3c1446893"}, +] + +[package.dependencies] +numpy = ">=1.14" + +[package.extras] +docs = ["matplotlib", "numpydoc (>=1.1.0,<1.2.0)", "sphinx", "sphinx-book-theme", "sphinx-remove-toctrees"] +test = ["pytest", "pytest-cov"] + [[package]] name = "six" version = "1.16.0" @@ -3462,6 +3860,36 @@ typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\"" [package.extras] full = ["httpx (>=0.22.0)", "itsdangerous", "jinja2", "python-multipart", "pyyaml"] +[[package]] +name = "svg-path" +version = "6.2" +description = "SVG path objects and parser" +category = "main" +optional = true +python-versions = "*" +files = [ + {file = "svg.path-6.2-py2.py3-none-any.whl", hash = "sha256:c3b12e6d372771b466078837252eb13b655ea2658437c426cc67fc6262433dc8"}, + {file = "svg.path-6.2.tar.gz", hash = "sha256:1a2159f9db898df93c4637cfd3ccaf7da1fd073f59fa9a5950c73e46d4aa1aca"}, +] + +[package.extras] +test = ["Pillow", "pytest", "pytest-cov"] + +[[package]] +name = "sympy" +version = "1.10.1" +description = "Computer algebra system (CAS) in Python" +category = "main" +optional = true +python-versions = ">=3.7" +files = [ + {file = "sympy-1.10.1-py3-none-any.whl", hash = "sha256:df75d738930f6fe9ebe7034e59d56698f29e85f443f743e51e47df0caccc2130"}, + {file = "sympy-1.10.1.tar.gz", hash = "sha256:5939eeffdf9e152172601463626c022a2c27e75cf6278de8d401d50c9d58787b"}, +] + +[package.dependencies] +mpmath = ">=0.19" + [[package]] name = "terminado" version = "0.17.0" @@ -3606,23 +4034,39 @@ test = ["pre-commit", "pytest"] [[package]] name = "trimesh" -version = "3.17.1" +version = "3.21.2" description = "Import, export, process, analyze and view triangular meshes." category = "main" optional = true python-versions = "*" files = [ - {file = "trimesh-3.17.1-py3-none-any.whl", hash = "sha256:a09460ee4e11c32bf9f0643b86241b3e3e2aa86296c4912a0738b76da3034c00"}, - {file = "trimesh-3.17.1.tar.gz", hash = "sha256:025bb2fa3a2e87bdd6873f11db45a7ca19216f2f8b6aed29140fca57e32c298e"}, + {file = "trimesh-3.21.2-py3-none-any.whl", hash = "sha256:8a575f2ee4dc77fc680aa1545c3503619027c75d09bcc4878d29d5f32e0476ee"}, + {file = "trimesh-3.21.2.tar.gz", hash = "sha256:5513c4fb540b286cb95bdf626b906e3cdb661ff7a85ee940a52f27f1275049a4"}, ] [package.dependencies] +chardet = {version = "*", optional = true, markers = "extra == \"easy\""} +colorlog = {version = "*", optional = true, markers = "extra == \"easy\""} +jsonschema = {version = "*", optional = true, markers = "extra == \"easy\""} +lxml = {version = "*", optional = true, markers = "extra == \"easy\""} +mapbox-earcut = {version = "*", optional = true, markers = "extra == \"easy\""} +networkx = {version = "*", optional = true, markers = "extra == \"easy\""} numpy = "*" +pillow = {version = "*", optional = true, markers = "extra == \"easy\""} +pycollada = {version = "*", optional = true, markers = "extra == \"easy\""} +requests = {version = "*", optional = true, markers = "extra == \"easy\""} +rtree = {version = "*", optional = true, markers = "extra == \"easy\""} +scipy = {version = "*", optional = true, markers = "extra == \"easy\""} +setuptools = {version = "*", optional = true, markers = "extra == \"easy\""} +shapely = {version = "*", optional = true, markers = "extra == \"easy\""} +"svg.path" = {version = "*", optional = true, markers = "extra == \"easy\""} +sympy = {version = "*", optional = true, markers = "extra == \"easy\""} +xxhash = {version = "*", optional = true, markers = "extra == \"easy\""} [package.extras] -all = ["chardet", "colorlog", "glooey", "jsonschema", "lxml", "mapbox-earcut", "meshio", "msgpack", "networkx", "pillow", "psutil", "pycollada", "pyglet (<2)", "python-fcl", "requests", "rtree", "scikit-image", "scipy", "setuptools", "shapely", "svg.path", "sympy", "xatlas", "xxhash"] -easy = ["chardet", "colorlog", "jsonschema", "lxml", "mapbox-earcut", "msgpack", "networkx", "pillow", "pycollada", "pyglet (<2)", "requests", "rtree", "scipy", "setuptools", "shapely", "svg.path", "sympy", "xxhash"] -test = ["coveralls", "ezdxf", "pyinstrument", "pytest", "pytest-cov"] +all = ["chardet", "colorlog", "glooey", "jsonschema", "lxml", "mapbox-earcut", "meshio", "networkx", "pillow", "psutil", "pycollada", "pyglet (<2)", "python-fcl", "requests", "rtree", "scikit-image", "scipy", "setuptools", "shapely", "svg.path", "sympy", "xatlas", "xxhash"] +easy = ["chardet", "colorlog", "jsonschema", "lxml", "mapbox-earcut", "networkx", "pillow", "pycollada", "requests", "rtree", "scipy", "setuptools", "shapely", "svg.path", "sympy", "xxhash"] +test = ["autopep8", "coveralls", "ezdxf", "pyinstrument", "pytest", "pytest-cov", "ruff"] [[package]] name = "typed-ast" @@ -3908,6 +4352,114 @@ files = [ [package.extras] test = ["pytest (>=3.0.0)"] +[[package]] +name = "xxhash" +version = "3.2.0" +description = "Python binding for xxHash" +category = "main" +optional = true +python-versions = ">=3.6" +files = [ + {file = "xxhash-3.2.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:af44b9e59c4b2926a4e3c7f9d29949ff42fcea28637ff6b8182e654461932be8"}, + {file = "xxhash-3.2.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1bdd57973e2b802ef32553d7bebf9402dac1557874dbe5c908b499ea917662cd"}, + {file = "xxhash-3.2.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b7c9aa77bbce61a5e681bd39cb6a804338474dcc90abe3c543592aa5d6c9a9b"}, + {file = "xxhash-3.2.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11bf87dc7bb8c3b0b5e24b7b941a9a19d8c1f88120b6a03a17264086bc8bb023"}, + {file = "xxhash-3.2.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2783d41487ce6d379fdfaa7332fca5187bf7010b9bddcf20cafba923bc1dc665"}, + {file = "xxhash-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:561076ca0dcef2fbc20b2bc2765bff099e002e96041ae9dbe910a863ca6ee3ea"}, + {file = "xxhash-3.2.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3a26eeb4625a6e61cedc8c1b39b89327c9c7e1a8c2c4d786fe3f178eb839ede6"}, + {file = "xxhash-3.2.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d93a44d0104d1b9b10de4e7aadf747f6efc1d7ec5ed0aa3f233a720725dd31bd"}, + {file = "xxhash-3.2.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:89585adc73395a10306d2e2036e50d6c4ac0cf8dd47edf914c25488871b64f6d"}, + {file = "xxhash-3.2.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:a892b4b139126a86bfdcb97cd912a2f8c4e8623869c3ef7b50871451dd7afeb0"}, + {file = "xxhash-3.2.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:e998efb190653f70e0f30d92b39fc645145369a4823bee46af8ddfc244aa969d"}, + {file = "xxhash-3.2.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e8ed3bd2b8bb3277710843ca63e4f5c3ee6f8f80b083be5b19a7a9905420d11e"}, + {file = "xxhash-3.2.0-cp310-cp310-win32.whl", hash = "sha256:20181cbaed033c72cb881b2a1d13c629cd1228f113046133469c9a48cfcbcd36"}, + {file = "xxhash-3.2.0-cp310-cp310-win_amd64.whl", hash = "sha256:a0f7a16138279d707db778a63264d1d6016ac13ffd3f1e99f54b2855d6c0d8e1"}, + {file = "xxhash-3.2.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5daff3fb5bfef30bc5a2cb143810d376d43461445aa17aece7210de52adbe151"}, + {file = "xxhash-3.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:75bb5be3c5de702a547715f320ecf5c8014aeca750ed5147ca75389bd22e7343"}, + {file = "xxhash-3.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01f36b671ff55cb1d5c2f6058b799b697fd0ae4b4582bba6ed0999678068172a"}, + {file = "xxhash-3.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d4d4519123aac73c93159eb8f61db9682393862dd669e7eae034ecd0a35eadac"}, + {file = "xxhash-3.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:994e4741d5ed70fc2a335a91ef79343c6b1089d7dfe6e955dd06f8ffe82bede6"}, + {file = "xxhash-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:919bc1b010aa6ff0eb918838ff73a435aed9e9a19c3202b91acecd296bf75607"}, + {file = "xxhash-3.2.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:17b65454c5accbb079c45eca546c27c4782f5175aa320758fafac896b1549d27"}, + {file = "xxhash-3.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b0c094d5e65a46dbf3fe0928ff20873a747e6abfd2ed4b675beeb2750624bc2e"}, + {file = "xxhash-3.2.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f94163ebe2d5546e6a5977e96d83621f4689c1054053428cf8d4c28b10f92f69"}, + {file = "xxhash-3.2.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cead7c0307977a00b3f784cff676e72c147adbcada19a2e6fc2ddf54f37cf387"}, + {file = "xxhash-3.2.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:a0e1bd0260c1da35c1883321ce2707ceea07127816ab625e1226ec95177b561a"}, + {file = "xxhash-3.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc8878935671490efe9275fb4190a6062b73277bd273237179b9b5a2aa436153"}, + {file = "xxhash-3.2.0-cp311-cp311-win32.whl", hash = "sha256:a433f6162b18d52f7068175d00bd5b1563b7405f926a48d888a97b90a160c40d"}, + {file = "xxhash-3.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:a32d546a1752e4ee7805d6db57944f7224afa7428d22867006b6486e4195c1f3"}, + {file = "xxhash-3.2.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:82daaab720866bf690b20b49de5640b0c27e3b8eea2d08aa75bdca2b0f0cfb63"}, + {file = "xxhash-3.2.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3126df6520cbdbaddd87ce74794b2b6c45dd2cf6ac2b600a374b8cdb76a2548c"}, + {file = "xxhash-3.2.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e172c1ee40507ae3b8d220f4048aaca204f203e1e4197e8e652f5c814f61d1aa"}, + {file = "xxhash-3.2.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5384f1d9f30876f5d5b618464fb19ff7ce6c0fe4c690fbaafd1c52adc3aae807"}, + {file = "xxhash-3.2.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26cb52174a7e96a17acad27a3ca65b24713610ac479c99ac9640843822d3bebf"}, + {file = "xxhash-3.2.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fbcd613a5e76b1495fc24db9c37a6b7ee5f214fd85979187ec4e032abfc12ded"}, + {file = "xxhash-3.2.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:f988daf25f31726d5b9d0be6af636ca9000898f9ea43a57eac594daea25b0948"}, + {file = "xxhash-3.2.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:bbc30c98ab006ab9fc47e5ed439c00f706bc9d4441ff52693b8b6fea335163e0"}, + {file = "xxhash-3.2.0-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:2408d49260b0a4a7cc6ba445aebf38e073aeaf482f8e32767ca477e32ccbbf9e"}, + {file = "xxhash-3.2.0-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:3f4152fd0bf8b03b79f2f900fd6087a66866537e94b5a11fd0fd99ef7efe5c42"}, + {file = "xxhash-3.2.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:0eea848758e4823a01abdbcccb021a03c1ee4100411cbeeb7a5c36a202a0c13c"}, + {file = "xxhash-3.2.0-cp36-cp36m-win32.whl", hash = "sha256:77709139af5123c578ab06cf999429cdb9ab211047acd0c787e098dcb3f1cb4d"}, + {file = "xxhash-3.2.0-cp36-cp36m-win_amd64.whl", hash = "sha256:91687671fd9d484a4e201ad266d366b695a45a1f2b41be93d116ba60f1b8f3b3"}, + {file = "xxhash-3.2.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e4af8bc5c3fcc2192c266421c6aa2daab1a18e002cb8e66ef672030e46ae25cf"}, + {file = "xxhash-3.2.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8be562e2ce3e481d9209b6f254c3d7c5ff920eb256aba2380d2fb5ba75d4f87"}, + {file = "xxhash-3.2.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9eba0c7c12126b12f7fcbea5513f28c950d28f33d2a227f74b50b77789e478e8"}, + {file = "xxhash-3.2.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2198c4901a0223c48f6ec0a978b60bca4f4f7229a11ca4dc96ca325dd6a29115"}, + {file = "xxhash-3.2.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50ce82a71b22a3069c02e914bf842118a53065e2ec1c6fb54786e03608ab89cc"}, + {file = "xxhash-3.2.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b5019fb33711c30e54e4e57ae0ca70af9d35b589d385ac04acd6954452fa73bb"}, + {file = "xxhash-3.2.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0d54ac023eef7e3ac9f0b8841ae8a376b933043bc2ad428121346c6fa61c491c"}, + {file = "xxhash-3.2.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c55fa832fc3fe64e0d29da5dc9b50ba66ca93312107cec2709300ea3d3bab5c7"}, + {file = "xxhash-3.2.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:f4ce006215497993ae77c612c1883ca4f3973899573ce0c52fee91f0d39c4561"}, + {file = "xxhash-3.2.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:1afb9b9d27fd675b436cb110c15979976d92d761ad6e66799b83756402f3a974"}, + {file = "xxhash-3.2.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:baa99cebf95c1885db21e119395f222a706a2bb75a545f0672880a442137725e"}, + {file = "xxhash-3.2.0-cp37-cp37m-win32.whl", hash = "sha256:75aa692936942ccb2e8fd6a386c81c61630ac1b6d6e921698122db8a930579c3"}, + {file = "xxhash-3.2.0-cp37-cp37m-win_amd64.whl", hash = "sha256:0a2cdfb5cae9fafb9f7b65fd52ecd60cf7d72c13bb2591ea59aaefa03d5a8827"}, + {file = "xxhash-3.2.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a68d1e8a390b660d94b9360ae5baa8c21a101bd9c4790a8b30781bada9f1fc6"}, + {file = "xxhash-3.2.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ce7c3ce28f94302df95eaea7c9c1e2c974b6d15d78a0c82142a97939d7b6c082"}, + {file = "xxhash-3.2.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0dcb419bf7b0bc77d366e5005c25682249c5521a63fd36c51f584bd91bb13bd5"}, + {file = "xxhash-3.2.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae521ed9287f86aac979eeac43af762f03d9d9797b2272185fb9ddd810391216"}, + {file = "xxhash-3.2.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b0d16775094423088ffa357d09fbbb9ab48d2fb721d42c0856b801c86f616eec"}, + {file = "xxhash-3.2.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe454aeab348c42f56d6f7434ff758a3ef90787ac81b9ad5a363cd61b90a1b0b"}, + {file = "xxhash-3.2.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:052fd0efdd5525c2dbc61bebb423d92aa619c4905bba605afbf1e985a562a231"}, + {file = "xxhash-3.2.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:02badf3754e2133de254a4688798c4d80f0060635087abcb461415cb3eb82115"}, + {file = "xxhash-3.2.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:66b8a90b28c13c2aae7a71b32638ceb14cefc2a1c8cf23d8d50dfb64dfac7aaf"}, + {file = "xxhash-3.2.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:649cdf19df175925ad87289ead6f760cd840730ee85abc5eb43be326a0a24d97"}, + {file = "xxhash-3.2.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:4b948a03f89f5c72d69d40975af8af241111f0643228796558dc1cae8f5560b0"}, + {file = "xxhash-3.2.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49f51fab7b762da7c2cee0a3d575184d3b9be5e2f64f26cae2dd286258ac9b3c"}, + {file = "xxhash-3.2.0-cp38-cp38-win32.whl", hash = "sha256:1a42994f0d42b55514785356722d9031f064fd34e495b3a589e96db68ee0179d"}, + {file = "xxhash-3.2.0-cp38-cp38-win_amd64.whl", hash = "sha256:0a6d58ba5865475e53d6c2c4fa6a62e2721e7875e146e2681e5337a6948f12e7"}, + {file = "xxhash-3.2.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:aabdbc082030f8df613e2d2ea1f974e7ad36a539bdfc40d36f34e55c7e4b8e94"}, + {file = "xxhash-3.2.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:498843b66b9ca416e9d03037e5875c8d0c0ab9037527e22df3b39aa5163214cd"}, + {file = "xxhash-3.2.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a910b1193cd90af17228f5d6069816646df0148f14f53eefa6b2b11a1dedfcd0"}, + {file = "xxhash-3.2.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb6d8ce31dc25faf4da92991320e211fa7f42de010ef51937b1dc565a4926501"}, + {file = "xxhash-3.2.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:883dc3d3942620f4c7dbc3fd6162f50a67f050b714e47da77444e3bcea7d91cc"}, + {file = "xxhash-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59dc8bfacf89b8f5be54d55bc3b4bd6d74d0c5320c8a63d2538ac7df5b96f1d5"}, + {file = "xxhash-3.2.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:61e6aa1d30c2af692aa88c4dd48709426e8b37bff6a574ee2de677579c34a3d6"}, + {file = "xxhash-3.2.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:314ec0bd21f0ee8d30f2bd82ed3759314bd317ddbbd8555668f3d20ab7a8899a"}, + {file = "xxhash-3.2.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:dad638cde3a5357ad3163b80b3127df61fb5b5e34e9e05a87697144400ba03c7"}, + {file = "xxhash-3.2.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:eaa3ea15025b56076d806b248948612289b093e8dcda8d013776b3848dffff15"}, + {file = "xxhash-3.2.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:7deae3a312feb5c17c97cbf18129f83cbd3f1f9ec25b0f50e2bd9697befb22e7"}, + {file = "xxhash-3.2.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:add774341c09853b1612c64a526032d95ab1683053325403e1afbe3ad2f374c5"}, + {file = "xxhash-3.2.0-cp39-cp39-win32.whl", hash = "sha256:9b94749130ef3119375c599bfce82142c2500ef9ed3280089157ee37662a7137"}, + {file = "xxhash-3.2.0-cp39-cp39-win_amd64.whl", hash = "sha256:e57d94a1552af67f67b27db5dba0b03783ea69d5ca2af2f40e098f0ba3ce3f5f"}, + {file = "xxhash-3.2.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:92fd765591c83e5c5f409b33eac1d3266c03d3d11c71a7dbade36d5cdee4fbc0"}, + {file = "xxhash-3.2.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8970f6a411a9839a02b23b7e90bbbba4a6de52ace009274998566dc43f36ca18"}, + {file = "xxhash-3.2.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5f3e33fe6cbab481727f9aeb136a213aed7e33cd1ca27bd75e916ffacc18411"}, + {file = "xxhash-3.2.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:368265392cb696dd53907e2328b5a8c1bee81cf2142d0cc743caf1c1047abb36"}, + {file = "xxhash-3.2.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:3b1f3c6d67fa9f49c4ff6b25ce0e7143bab88a5bc0f4116dd290c92337d0ecc7"}, + {file = "xxhash-3.2.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:c5e8db6e1ee7267b7c412ad0afd5863bf7a95286b8333a5958c8097c69f94cf5"}, + {file = "xxhash-3.2.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:761df3c7e2c5270088b691c5a8121004f84318177da1ca1db64222ec83c44871"}, + {file = "xxhash-3.2.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2d15a707e7f689531eb4134eccb0f8bf3844bb8255ad50823aa39708d9e6755"}, + {file = "xxhash-3.2.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e6b2ba4ff53dd5f57d728095e3def7375eb19c90621ce3b41b256de84ec61cfd"}, + {file = "xxhash-3.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:61b0bcf946fdfd8ab5f09179dc2b5c74d1ef47cedfc6ed0ec01fdf0ee8682dd3"}, + {file = "xxhash-3.2.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:f7b79f0f302396d8e0d444826ceb3d07b61977793886ebae04e82796c02e42dc"}, + {file = "xxhash-3.2.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e0773cd5c438ffcd5dbff91cdd503574f88a4b960e70cedeb67736583a17a918"}, + {file = "xxhash-3.2.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ec1f57127879b419a2c8d2db9d9978eb26c61ae17e5972197830430ae78d25b"}, + {file = "xxhash-3.2.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3d4b15c00e807b1d3d0b612338c814739dec310b80fb069bd732b98ddc709ad7"}, + {file = "xxhash-3.2.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:9d3f686e3d1c8900c5459eee02b60c7399e20ec5c6402364068a343c83a61d90"}, + {file = "xxhash-3.2.0.tar.gz", hash = "sha256:1afd47af8955c5db730f630ad53ae798cf7fae0acb64cebb3cf94d35c47dd088"}, +] + [[package]] name = "yarl" version = "1.8.2" @@ -4029,4 +4581,4 @@ web = ["fastapi"] [metadata] lock-version = "2.0" python-versions = ">=3.7,<4.0" -content-hash = "3c0b9daa397c3f62d87970839a0d5cb84ddba0e96bd150dfa60c4c15d03fd1c3" +content-hash = "bc2ee0a24999257a7d6f76cb120864a4aac2426e24999e299eeea273b895c8c9" diff --git a/pyproject.toml b/pyproject.toml index 08030bf557d..e5c394ac39e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ torch = { version = ">=1.0.0", optional = true } orjson = ">=3.8.2" pillow = {version = ">=9.3.0", optional = true } types-pillow = {version = ">=9.3.0.1", optional = true } -trimesh = {version = ">=3.17.1", optional = true} +trimesh = {version = ">=3.17.1", extras = ["easy"], optional = true } typing-inspect = ">=0.8.0" types-requests = ">=2.28.11.6" av = {version = ">=10.0.0", optional = true} From d6605e280f3b296d288f7532fcbfc7b4856c3c4f Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 11:28:01 +0200 Subject: [PATCH 33/66] fix: pil immage importfix: clean up Signed-off-by: anna-charlotte --- docarray/typing/bytes/image_bytes.py | 3 ++- .../tensor/image/abstract_image_tensor.py | 6 +++-- docarray/utils/_internal/compress.py | 23 ++++++++++--------- docarray/utils/_internal/misc.py | 2 +- 4 files changed, 19 insertions(+), 15 deletions(-) diff --git a/docarray/typing/bytes/image_bytes.py b/docarray/typing/bytes/image_bytes.py index 2ab187d12f5..d21dbe8af9e 100644 --- a/docarray/typing/bytes/image_bytes.py +++ b/docarray/typing/bytes/image_bytes.py @@ -91,7 +91,8 @@ class MyDoc(BaseDoc): if TYPE_CHECKING: from PIL import Image as PILImage else: - PILImage = import_library('PIL', raise_error=True).Image + PIL = import_library('PIL', raise_error=True) # noqa: F841 + from PIL import Image as PILImage raw_img = PILImage.open(BytesIO(self)) if width or height: diff --git a/docarray/typing/tensor/image/abstract_image_tensor.py b/docarray/typing/tensor/image/abstract_image_tensor.py index fd262e4cdc5..7b033a4ee33 100644 --- a/docarray/typing/tensor/image/abstract_image_tensor.py +++ b/docarray/typing/tensor/image/abstract_image_tensor.py @@ -18,7 +18,8 @@ def to_bytes(self, format: str = 'PNG') -> bytes: if TYPE_CHECKING: from PIL import Image as PILImage else: - PILImage = import_library('PIL', raise_error=True).Image + PIL = import_library('PIL', raise_error=True) # noqa: F841 + from PIL import Image as PILImage if format == 'jpg': format = 'jpeg' # unify it to ISO standard @@ -42,7 +43,8 @@ def display(self) -> None: if TYPE_CHECKING: from PIL import Image as PILImage else: - PILImage = import_library('PIL', raise_error=True).Image + PIL = import_library('PIL', raise_error=True) # noqa: F841 + from PIL import Image as PILImage np_array = self.get_comp_backend().to_numpy(self) img = PILImage.fromarray(np_array) diff --git a/docarray/utils/_internal/compress.py b/docarray/utils/_internal/compress.py index ab37ad1a694..51b4860fe4a 100644 --- a/docarray/utils/_internal/compress.py +++ b/docarray/utils/_internal/compress.py @@ -6,11 +6,12 @@ def _compress_bytes(data: bytes, algorithm: Optional[str] = None) -> bytes: if algorithm == 'lz4': if TYPE_CHECKING: - import lz4.frame as lz4_frame # type: ignore + from lz4 import frame else: - lz4_frame = import_library('lz4', raise_error=True).frame + lz4 = import_library('lz4', raise_error=True) # noqa: F841 + from lz4 import frame - data = lz4_frame.compress(data) + data = frame.compress(data) elif algorithm == 'bz2': import bz2 @@ -32,13 +33,13 @@ def _compress_bytes(data: bytes, algorithm: Optional[str] = None) -> bytes: def _decompress_bytes(data: bytes, algorithm: Optional[str] = None) -> bytes: if algorithm == 'lz4': - if TYPE_CHECKING: - import lz4.frame as lz4_frame # type: ignore + from lz4 import frame else: - lz4_frame = import_library('lz4', raise_error=True).frame + lz4 = import_library('lz4', raise_error=True) # noqa: F841 + from lz4 import frame - data = lz4_frame.decompress(data) + data = frame.decompress(data) elif algorithm == 'bz2': import bz2 @@ -60,14 +61,14 @@ def _decompress_bytes(data: bytes, algorithm: Optional[str] = None) -> bytes: def _get_compress_ctx(algorithm: Optional[str] = None) -> Optional[Callable]: if algorithm == 'lz4': - if TYPE_CHECKING: - import lz4.frame as lz4_frame # type: ignore + from lz4 import frame else: - lz4_frame = import_library('lz4', raise_error=True).frame + lz4 = import_library('lz4', raise_error=True) # noqa: F841 + from lz4 import frame def _fun(x: IO[bytes]): - return lz4_frame.LZ4FrameFile(x, 'wb') + return frame.LZ4FrameFile(x, 'wb') compress_ctx = _fun elif algorithm == 'gzip': diff --git a/docarray/utils/_internal/misc.py b/docarray/utils/_internal/misc.py index 72601e9f57c..079887ddead 100644 --- a/docarray/utils/_internal/misc.py +++ b/docarray/utils/_internal/misc.py @@ -50,7 +50,7 @@ def import_library( if lib is None and raise_error: raise RuntimeError( - f'The following required library is not installed: {package} ' + f'The following required library is not installed: {package} \n' f'To install all necessary libraries, run: `pip install {INSTALL_INSTRUCTIONS[package]}`.' ) else: From a368483ecb8e7e61479ed82ffe9860696f360227 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 11:34:41 +0200 Subject: [PATCH 34/66] chore: add lz4 to mypy missing type hint section Signed-off-by: anna-charlotte --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index e5c394ac39e..4191d06a588 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,6 +117,10 @@ ignore_missing_imports = true module = "pydub" ignore_missing_imports = true +[[tool.mypy.overrides]] +module = "lz4" +ignore_missing_imports = true + [tool.black] skip-string-normalization = true # equivalent to black -S exclude = 'docarray/proto/pb*/*' From 1246a4d9f889bb2bbfda82ddabd0ed115276edb9 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 12:13:19 +0200 Subject: [PATCH 35/66] docs: add instructions to doc index tutorial Signed-off-by: anna-charlotte --- docs/how_to/add_doc_index.md | 49 +++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/docs/how_to/add_doc_index.md b/docs/how_to/add_doc_index.md index 8fb03b9978b..054a2b94cef 100644 --- a/docs/how_to/add_doc_index.md +++ b/docs/how_to/add_doc_index.md @@ -9,10 +9,11 @@ This document shows how to add a new Document Index to DocArray. That process can be broken down into a number of basic steps: -1. Create a new class that inherits from `BaseDocIndex` -2. Declare default configurations for your Document Index -3. Implement abstract methods for indexing, searching, and deleting -4. Implement a Query Builder for your Document Index +1. Installation and user instructions +2. Create a new class that inherits from `BaseDocIndex`Create a new class that inherits from `BaseDocIndex` +3. Declare default configurations for your Document Index +4. Implement abstract methods for indexing, searching, and deleting +5. Implement a Query Builder for your Document Index In general, the steps above can be followed in roughly that order. @@ -25,6 +26,46 @@ For an end-to-end example of this process, you can check out the [existing HNSWL For example, HNSWLib can only index vectors, so it uses SQLite to store the rest of the Documents alongside it. This is _not_ how you should store Documents in your implementation! You can find guidance on how you _should_ do it below. + +## Installation and user instructions +Add the library required for your Index via poetry: `poetry add {my_index_lib}`. +In the `pyproject.toml` file, it will look like this: +``` +[tool.poetry.dependencies] +my_index_lib = ">=123.456.789" +``` +Mark it as optional and manually create an extra for it: +``` +[tool.poetry.dependencies] +my_index_lib = {version = ">=0.6.2", optional = true } + +[tool.poetry.extras] +my_index_extra = ["my_index_lib"] +``` + +In case the user tries to use your Index without the correct installs, we want to throw an error with corresponding instructions. + +To enable this, first, add instructions to the `INSTALL_INSTRUCTIONS` dictionary in `docarray/utils/misc.py`, such as +```python +{'my_index_lib': '"docarray[my_index_extra]"'} +``` +Next, ensure to add a case to the `__getattr__()` for your new Index to `docarray/index/__init__.py`. By doing so, the user will be given the instructions, when trying to import `MyIndex` without the correct libraries installed. + +```python +if TYPE_CHECKING: + from docarray.index.backends.my_index import MyIndex # noqa: F401 + + +def __getattr__(name: str): + if name == 'HnswDocumentIndex': + import_library('hnswlib', raise_error=True) + from docarray.index.backends.my_index import MyIndex # noqa + + __all__.append('MyIndex') + return MyIndex +``` +Additionally, wrap the required imports in the file where the `MyIndex` class will be located, such as it was done in `docarray/index/backends/hnswlib.py`. + ## Create a new Document Index class To get started, create a new class that inherits from `BaseDocIndex` and `typing.Generic`: From 34ef472e87e0a3ae7be7c0e86fb2e0a62ce34b49 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 13:18:21 +0200 Subject: [PATCH 36/66] chore: extra pandas and condense module where missing imports ignore Signed-off-by: anna-charlotte --- docarray/utils/_internal/misc.py | 2 +- pyproject.toml | 62 +++++++++----------------------- 2 files changed, 17 insertions(+), 47 deletions(-) diff --git a/docarray/utils/_internal/misc.py b/docarray/utils/_internal/misc.py index 079887ddead..e3262cd067e 100644 --- a/docarray/utils/_internal/misc.py +++ b/docarray/utils/_internal/misc.py @@ -21,9 +21,9 @@ INSTALL_INSTRUCTIONS = { - 'pandas': '"docarray[common]"', 'google.protobuf': '"docarray[common]"', 'lz4': '"docarray[common]"', + 'pandas': '"docarray[pandas]"', 'PIL': '"docarray[image]"', 'pydub': '"docarray[audio]"', 'av': '"docarray[video]"', diff --git a/pyproject.toml b/pyproject.toml index 4191d06a588..f8accde29e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,7 +28,8 @@ smart-open = {version = ">=6.3.0", extras = ["s3"], optional = true} jina-hubble-sdk = {version = ">=0.34.0", optional = true} [tool.poetry.extras] -common = ["protobuf", "lz4", "pandas"] +common = ["protobuf", "lz4"] +pandas = ["pandas"] image = ["pillow", "types-pillow"] video = ["av"] audio = ["pydub"] @@ -74,51 +75,20 @@ plugins = "pydantic.mypy" check_untyped_defs = true [[tool.mypy.overrides]] -module = "av" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pandas" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "trimesh" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "hnswlib" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "typing_inspect" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "IPython.display" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "hubble.*" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "smart_open" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "boto3" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "botocore" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "pydub" -ignore_missing_imports = true - -[[tool.mypy.overrides]] -module = "lz4" +module = [ + "botocore", + "pydub", + "boto3", + "lz4", + "smart_open", + "hubble.*", + "IPython.display", + "typing_inspect", + "hnswlib", + "trimesh", + "pandas", + "av", +] ignore_missing_imports = true [tool.black] From b26bd146c44c38e1252d2d30f3b661bd05714f54 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 13:19:56 +0200 Subject: [PATCH 37/66] fix: update poetry lock Signed-off-by: anna-charlotte --- poetry.lock | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index f8e9abbb543..473da526ef1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4568,12 +4568,13 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [extras] audio = ["pydub"] aws = ["smart-open"] -common = ["protobuf", "lz4", "pandas"] +common = ["protobuf", "lz4"] full = ["protobuf", "lz4", "pandas", "pillow", "types-pillow", "av", "pydub", "trimesh"] hnswlib = ["hnswlib"] image = ["pillow", "types-pillow"] jac = ["jina-hubble-sdk"] mesh = ["trimesh"] +pandas = ["pandas"] torch = ["torch"] video = ["av"] web = ["fastapi"] @@ -4581,4 +4582,4 @@ web = ["fastapi"] [metadata] lock-version = "2.0" python-versions = ">=3.7,<4.0" -content-hash = "bc2ee0a24999257a7d6f76cb120864a4aac2426e24999e299eeea273b895c8c9" +content-hash = "33f01d36bd1b134d7ab90505bba21a95a500d0898aeaae63e71e8213616098c2" From 280a5cd1109cb5753edc03f57eab97cc61d5e137 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 14:03:00 +0200 Subject: [PATCH 38/66] fix: missed imports Signed-off-by: anna-charlotte --- docarray/base_doc/__init__.py | 2 +- docarray/base_doc/doc_response.py | 2 +- docarray/computation/tensorflow_backend.py | 2 +- docarray/computation/torch_backend.py | 2 +- docarray/documents/mesh/vertices_and_faces.py | 2 +- docarray/index/__init__.py | 2 +- docarray/proto/__init__.py | 2 +- docarray/store/__init__.py | 2 +- docarray/typing/bytes/audio_bytes.py | 2 +- docarray/typing/bytes/image_bytes.py | 2 +- docarray/typing/bytes/video_bytes.py | 2 +- docarray/typing/tensor/tensorflow_tensor.py | 2 +- docarray/typing/tensor/torch_tensor.py | 2 +- docarray/typing/url/url_3d/url_3d.py | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docarray/base_doc/__init__.py b/docarray/base_doc/__init__.py index 9e145403066..489ed0015a0 100644 --- a/docarray/base_doc/__init__.py +++ b/docarray/base_doc/__init__.py @@ -4,7 +4,7 @@ __all__ = ['AnyDoc', 'BaseDoc', 'BaseNode'] -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library def __getattr__(name: str): diff --git a/docarray/base_doc/doc_response.py b/docarray/base_doc/doc_response.py index f0a1686a4a8..7d2b7b76ef6 100644 --- a/docarray/base_doc/doc_response.py +++ b/docarray/base_doc/doc_response.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING, Any -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from fastapi.responses import JSONResponse diff --git a/docarray/computation/tensorflow_backend.py b/docarray/computation/tensorflow_backend.py index 41341e3066b..fc963cdb48b 100644 --- a/docarray/computation/tensorflow_backend.py +++ b/docarray/computation/tensorflow_backend.py @@ -6,7 +6,7 @@ from docarray.computation import AbstractComputationalBackend from docarray.computation.abstract_numpy_based_backend import AbstractNumpyBasedBackend from docarray.typing import TensorFlowTensor -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: import tensorflow as tf # type: ignore diff --git a/docarray/computation/torch_backend.py b/docarray/computation/torch_backend.py index 0871fc2e689..f64b993186a 100644 --- a/docarray/computation/torch_backend.py +++ b/docarray/computation/torch_backend.py @@ -3,7 +3,7 @@ import numpy as np from docarray.computation.abstract_comp_backend import AbstractComputationalBackend -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: import torch diff --git a/docarray/documents/mesh/vertices_and_faces.py b/docarray/documents/mesh/vertices_and_faces.py index 19697b1aa98..cc3dc6da795 100644 --- a/docarray/documents/mesh/vertices_and_faces.py +++ b/docarray/documents/mesh/vertices_and_faces.py @@ -2,7 +2,7 @@ from docarray.base_doc import BaseDoc from docarray.typing.tensor.tensor import AnyTensor -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library T = TypeVar('T', bound='VerticesAndFaces') diff --git a/docarray/index/__init__.py b/docarray/index/__init__.py index 1f6f8af8aca..55a3e8dd318 100644 --- a/docarray/index/__init__.py +++ b/docarray/index/__init__.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library __all__ = [] diff --git a/docarray/proto/__init__.py b/docarray/proto/__init__.py index bca72030b79..163809b7ef0 100644 --- a/docarray/proto/__init__.py +++ b/docarray/proto/__init__.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from google.protobuf import __version__ as __pb__version__ diff --git a/docarray/store/__init__.py b/docarray/store/__init__.py index 1456c9986ab..c1174042e41 100644 --- a/docarray/store/__init__.py +++ b/docarray/store/__init__.py @@ -5,7 +5,7 @@ __all__ = ['FileDocStore'] -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from docarray.store.jac import JACDocStore # noqa: F401 diff --git a/docarray/typing/bytes/audio_bytes.py b/docarray/typing/bytes/audio_bytes.py index aae0a628d35..643c753067c 100644 --- a/docarray/typing/bytes/audio_bytes.py +++ b/docarray/typing/bytes/audio_bytes.py @@ -7,7 +7,7 @@ from docarray.typing.abstract_type import AbstractType from docarray.typing.proto_register import _register_proto -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from pydantic.fields import BaseConfig, ModelField diff --git a/docarray/typing/bytes/image_bytes.py b/docarray/typing/bytes/image_bytes.py index d21dbe8af9e..c7d2bf8401d 100644 --- a/docarray/typing/bytes/image_bytes.py +++ b/docarray/typing/bytes/image_bytes.py @@ -7,7 +7,7 @@ from docarray.typing.abstract_type import AbstractType from docarray.typing.proto_register import _register_proto -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from pydantic.fields import BaseConfig, ModelField diff --git a/docarray/typing/bytes/video_bytes.py b/docarray/typing/bytes/video_bytes.py index 079924b4a87..31b6cdc97a4 100644 --- a/docarray/typing/bytes/video_bytes.py +++ b/docarray/typing/bytes/video_bytes.py @@ -8,7 +8,7 @@ from docarray.typing import AudioNdArray, NdArray, VideoNdArray from docarray.typing.abstract_type import AbstractType from docarray.typing.proto_register import _register_proto -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from pydantic.fields import BaseConfig, ModelField diff --git a/docarray/typing/tensor/tensorflow_tensor.py b/docarray/typing/tensor/tensorflow_tensor.py index 2329a8c689a..f0d79aa0b30 100644 --- a/docarray/typing/tensor/tensorflow_tensor.py +++ b/docarray/typing/tensor/tensorflow_tensor.py @@ -5,7 +5,7 @@ from docarray.base_doc.base_node import BaseNode from docarray.typing.proto_register import _register_proto from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: import tensorflow as tf # type: ignore diff --git a/docarray/typing/tensor/torch_tensor.py b/docarray/typing/tensor/torch_tensor.py index eb62cff2895..e2ece8c84dd 100644 --- a/docarray/typing/tensor/torch_tensor.py +++ b/docarray/typing/tensor/torch_tensor.py @@ -6,7 +6,7 @@ from docarray.base_doc.base_node import BaseNode from docarray.typing.proto_register import _register_proto from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: import torch diff --git a/docarray/typing/url/url_3d/url_3d.py b/docarray/typing/url/url_3d/url_3d.py index b4349251131..b6d77b60829 100644 --- a/docarray/typing/url/url_3d/url_3d.py +++ b/docarray/typing/url/url_3d/url_3d.py @@ -3,7 +3,7 @@ from docarray.typing.proto_register import _register_proto from docarray.typing.url.any_url import AnyUrl -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: import trimesh From 33f78e99afc4b76756110c5dcb142e8f326268e7 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 14:54:37 +0200 Subject: [PATCH 39/66] fix: clean up Signed-off-by: anna-charlotte --- docarray/base_doc/__init__.py | 3 +-- docarray/index/__init__.py | 4 +-- docarray/store/__init__.py | 6 ++--- docarray/typing/__init__.py | 26 +++++++++----------- docarray/typing/tensor/__init__.py | 19 +++++++------- docarray/typing/tensor/audio/__init__.py | 5 ++-- docarray/typing/tensor/embedding/__init__.py | 5 ++-- docarray/typing/tensor/image/__init__.py | 5 ++-- docarray/typing/tensor/video/__init__.py | 5 ++-- 9 files changed, 34 insertions(+), 44 deletions(-) diff --git a/docarray/base_doc/__init__.py b/docarray/base_doc/__init__.py index 489ed0015a0..71e7bb53f7d 100644 --- a/docarray/base_doc/__init__.py +++ b/docarray/base_doc/__init__.py @@ -1,11 +1,10 @@ from docarray.base_doc.any_doc import AnyDoc from docarray.base_doc.base_node import BaseNode from docarray.base_doc.doc import BaseDoc +from docarray.utils._internal.misc import import_library __all__ = ['AnyDoc', 'BaseDoc', 'BaseNode'] -from docarray.utils._internal.misc import import_library - def __getattr__(name: str): if name == 'DocResponse': diff --git a/docarray/index/__init__.py b/docarray/index/__init__.py index 55a3e8dd318..113853df72c 100644 --- a/docarray/index/__init__.py +++ b/docarray/index/__init__.py @@ -2,11 +2,11 @@ from docarray.utils._internal.misc import import_library -__all__ = [] - if TYPE_CHECKING: from docarray.index.backends.hnswlib import HnswDocumentIndex # noqa: F401 +__all__ = [] + def __getattr__(name: str): if name == 'HnswDocumentIndex': diff --git a/docarray/store/__init__.py b/docarray/store/__init__.py index c1174042e41..11275704d2e 100644 --- a/docarray/store/__init__.py +++ b/docarray/store/__init__.py @@ -1,16 +1,14 @@ from typing import TYPE_CHECKING from docarray.store.file import FileDocStore - -__all__ = ['FileDocStore'] - - from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from docarray.store.jac import JACDocStore # noqa: F401 from docarray.store.s3 import S3DocStore # noqa: F401 +__all__ = ['FileDocStore'] + def __getattr__(name: str): if name == 'JACDocStore': diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index 1db60181415..66ea3496c97 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -17,6 +17,18 @@ TextUrl, VideoUrl, ) +from docarray.utils._internal.misc import import_library + +if TYPE_CHECKING: + from docarray.typing.tensor import TensorFlowTensor # noqa: F401 + from docarray.typing.tensor import TorchEmbedding, TorchTensor # noqa: F401 + from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F401 + from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 + from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 + from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.video import VideoTorchTensor # noqa: F401 __all__ = [ 'NdArray', @@ -39,20 +51,6 @@ 'ImageNdArray', ] -from docarray.utils._internal.misc import import_library - -if TYPE_CHECKING: - from docarray.typing.tensor import TensorFlowTensor # noqa: F401 - from docarray.typing.tensor import TorchEmbedding, TorchTensor # noqa: F401 - from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F401 - from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 - from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 - from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.video import VideoTorchTensor # noqa: F401 - - torch_tensors = [ 'AudioTorchTensor', 'TorchEmbedding', diff --git a/docarray/typing/tensor/__init__.py b/docarray/typing/tensor/__init__.py index c40feac93b8..6e5b7e6e474 100644 --- a/docarray/typing/tensor/__init__.py +++ b/docarray/typing/tensor/__init__.py @@ -4,16 +4,6 @@ from docarray.typing.tensor.image import ImageNdArray, ImageTensor from docarray.typing.tensor.ndarray import NdArray from docarray.typing.tensor.tensor import AnyTensor - -__all__ = [ - 'NdArray', - 'AnyTensor', - 'AnyEmbedding', - 'NdArrayEmbedding', - 'ImageNdArray', - 'ImageTensor', -] - from docarray.utils._internal.misc import import_library if TYPE_CHECKING: @@ -24,6 +14,15 @@ from docarray.typing.tensor.tensorflow_tensor import TensorFlowTensor # noqa: F401 from docarray.typing.tensor.torch_tensor import TorchTensor # noqa: F401 +__all__ = [ + 'NdArray', + 'AnyTensor', + 'AnyEmbedding', + 'NdArrayEmbedding', + 'ImageNdArray', + 'ImageTensor', +] + torch_tensors = ['TorchEmbedding', 'ImageTorchTensor', 'TorchTensor'] tf_tensors = ['TensorFlowEmbedding', 'TensorFlowTensor', 'ImageTensorFlowTensor'] diff --git a/docarray/typing/tensor/audio/__init__.py b/docarray/typing/tensor/audio/__init__.py index 30973a7ad12..38baf0ff240 100644 --- a/docarray/typing/tensor/audio/__init__.py +++ b/docarray/typing/tensor/audio/__init__.py @@ -2,9 +2,6 @@ from docarray.typing.tensor.audio.audio_ndarray import AudioNdArray from docarray.typing.tensor.audio.audio_tensor import AudioTensor - -__all__ = ['AudioNdArray', 'AudioTensor'] - from docarray.utils._internal.misc import import_library if TYPE_CHECKING: @@ -13,6 +10,8 @@ ) from docarray.typing.tensor.audio.audio_torch_tensor import AudioTorchTensor # noqa +__all__ = ['AudioNdArray', 'AudioTensor'] + def __getattr__(name: str): if name == 'AudioTorchTensor': diff --git a/docarray/typing/tensor/embedding/__init__.py b/docarray/typing/tensor/embedding/__init__.py index 42814851999..aabe4272107 100644 --- a/docarray/typing/tensor/embedding/__init__.py +++ b/docarray/typing/tensor/embedding/__init__.py @@ -2,15 +2,14 @@ from docarray.typing.tensor.embedding.embedding import AnyEmbedding from docarray.typing.tensor.embedding.ndarray import NdArrayEmbedding - -__all__ = ['NdArrayEmbedding', 'AnyEmbedding'] - from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from docarray.typing.tensor.embedding.tensorflow import TensorFlowEmbedding # noqa from docarray.typing.tensor.embedding.torch import TorchEmbedding # noqa +__all__ = ['NdArrayEmbedding', 'AnyEmbedding'] + def __getattr__(name: str): if name == 'TorchEmbedding': diff --git a/docarray/typing/tensor/image/__init__.py b/docarray/typing/tensor/image/__init__.py index 1c199a5ff81..6dd56cc161c 100644 --- a/docarray/typing/tensor/image/__init__.py +++ b/docarray/typing/tensor/image/__init__.py @@ -2,9 +2,6 @@ from docarray.typing.tensor.image.image_ndarray import ImageNdArray from docarray.typing.tensor.image.image_tensor import ImageTensor - -__all__ = ['ImageNdArray', 'ImageTensor'] - from docarray.utils._internal.misc import import_library if TYPE_CHECKING: @@ -13,6 +10,8 @@ ) from docarray.typing.tensor.image.image_torch_tensor import ImageTorchTensor # noqa +__all__ = ['ImageNdArray', 'ImageTensor'] + def __getattr__(name: str): if name == 'ImageTorchTensor': diff --git a/docarray/typing/tensor/video/__init__.py b/docarray/typing/tensor/video/__init__.py index 8262e30f357..f495a22cc13 100644 --- a/docarray/typing/tensor/video/__init__.py +++ b/docarray/typing/tensor/video/__init__.py @@ -2,9 +2,6 @@ from docarray.typing.tensor.video.video_ndarray import VideoNdArray from docarray.typing.tensor.video.video_tensor import VideoTensor - -__all__ = ['VideoNdArray', 'VideoTensor'] - from docarray.utils._internal.misc import import_library if TYPE_CHECKING: @@ -13,6 +10,8 @@ ) from docarray.typing.tensor.video.video_torch_tensor import VideoTorchTensor # noqa +__all__ = ['VideoNdArray', 'VideoTensor'] + def __getattr__(name: str): if name == 'VideoTorchTensor': From a725c3b2c7adc496e4a05c79da430eaf67416d66 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 15:20:59 +0200 Subject: [PATCH 40/66] fix: revert last commit This reverts commit 9aca06f60ba5bc32b91271a7743d3869ac882434. Signed-off-by: anna-charlotte --- docarray/base_doc/__init__.py | 3 ++- docarray/index/__init__.py | 4 +-- docarray/store/__init__.py | 6 +++-- docarray/typing/__init__.py | 26 +++++++++++--------- docarray/typing/tensor/__init__.py | 19 +++++++------- docarray/typing/tensor/audio/__init__.py | 5 ++-- docarray/typing/tensor/embedding/__init__.py | 5 ++-- docarray/typing/tensor/image/__init__.py | 5 ++-- docarray/typing/tensor/video/__init__.py | 5 ++-- 9 files changed, 44 insertions(+), 34 deletions(-) diff --git a/docarray/base_doc/__init__.py b/docarray/base_doc/__init__.py index 71e7bb53f7d..489ed0015a0 100644 --- a/docarray/base_doc/__init__.py +++ b/docarray/base_doc/__init__.py @@ -1,10 +1,11 @@ from docarray.base_doc.any_doc import AnyDoc from docarray.base_doc.base_node import BaseNode from docarray.base_doc.doc import BaseDoc -from docarray.utils._internal.misc import import_library __all__ = ['AnyDoc', 'BaseDoc', 'BaseNode'] +from docarray.utils._internal.misc import import_library + def __getattr__(name: str): if name == 'DocResponse': diff --git a/docarray/index/__init__.py b/docarray/index/__init__.py index 113853df72c..55a3e8dd318 100644 --- a/docarray/index/__init__.py +++ b/docarray/index/__init__.py @@ -2,11 +2,11 @@ from docarray.utils._internal.misc import import_library +__all__ = [] + if TYPE_CHECKING: from docarray.index.backends.hnswlib import HnswDocumentIndex # noqa: F401 -__all__ = [] - def __getattr__(name: str): if name == 'HnswDocumentIndex': diff --git a/docarray/store/__init__.py b/docarray/store/__init__.py index 11275704d2e..c1174042e41 100644 --- a/docarray/store/__init__.py +++ b/docarray/store/__init__.py @@ -1,14 +1,16 @@ from typing import TYPE_CHECKING from docarray.store.file import FileDocStore + +__all__ = ['FileDocStore'] + + from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from docarray.store.jac import JACDocStore # noqa: F401 from docarray.store.s3 import S3DocStore # noqa: F401 -__all__ = ['FileDocStore'] - def __getattr__(name: str): if name == 'JACDocStore': diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index 66ea3496c97..1db60181415 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -17,18 +17,6 @@ TextUrl, VideoUrl, ) -from docarray.utils._internal.misc import import_library - -if TYPE_CHECKING: - from docarray.typing.tensor import TensorFlowTensor # noqa: F401 - from docarray.typing.tensor import TorchEmbedding, TorchTensor # noqa: F401 - from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F401 - from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 - from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 - from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.video import VideoTorchTensor # noqa: F401 __all__ = [ 'NdArray', @@ -51,6 +39,20 @@ 'ImageNdArray', ] +from docarray.utils._internal.misc import import_library + +if TYPE_CHECKING: + from docarray.typing.tensor import TensorFlowTensor # noqa: F401 + from docarray.typing.tensor import TorchEmbedding, TorchTensor # noqa: F401 + from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F401 + from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 + from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 + from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.video import VideoTorchTensor # noqa: F401 + + torch_tensors = [ 'AudioTorchTensor', 'TorchEmbedding', diff --git a/docarray/typing/tensor/__init__.py b/docarray/typing/tensor/__init__.py index 6e5b7e6e474..c40feac93b8 100644 --- a/docarray/typing/tensor/__init__.py +++ b/docarray/typing/tensor/__init__.py @@ -4,15 +4,6 @@ from docarray.typing.tensor.image import ImageNdArray, ImageTensor from docarray.typing.tensor.ndarray import NdArray from docarray.typing.tensor.tensor import AnyTensor -from docarray.utils._internal.misc import import_library - -if TYPE_CHECKING: - from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 - from docarray.typing.tensor.embedding import TorchEmbedding # noqa: F401 - from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 - from docarray.typing.tensor.tensorflow_tensor import TensorFlowTensor # noqa: F401 - from docarray.typing.tensor.torch_tensor import TorchTensor # noqa: F401 __all__ = [ 'NdArray', @@ -23,6 +14,16 @@ 'ImageTensor', ] +from docarray.utils._internal.misc import import_library + +if TYPE_CHECKING: + from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 + from docarray.typing.tensor.embedding import TorchEmbedding # noqa: F401 + from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 + from docarray.typing.tensor.tensorflow_tensor import TensorFlowTensor # noqa: F401 + from docarray.typing.tensor.torch_tensor import TorchTensor # noqa: F401 + torch_tensors = ['TorchEmbedding', 'ImageTorchTensor', 'TorchTensor'] tf_tensors = ['TensorFlowEmbedding', 'TensorFlowTensor', 'ImageTensorFlowTensor'] diff --git a/docarray/typing/tensor/audio/__init__.py b/docarray/typing/tensor/audio/__init__.py index 38baf0ff240..30973a7ad12 100644 --- a/docarray/typing/tensor/audio/__init__.py +++ b/docarray/typing/tensor/audio/__init__.py @@ -2,6 +2,9 @@ from docarray.typing.tensor.audio.audio_ndarray import AudioNdArray from docarray.typing.tensor.audio.audio_tensor import AudioTensor + +__all__ = ['AudioNdArray', 'AudioTensor'] + from docarray.utils._internal.misc import import_library if TYPE_CHECKING: @@ -10,8 +13,6 @@ ) from docarray.typing.tensor.audio.audio_torch_tensor import AudioTorchTensor # noqa -__all__ = ['AudioNdArray', 'AudioTensor'] - def __getattr__(name: str): if name == 'AudioTorchTensor': diff --git a/docarray/typing/tensor/embedding/__init__.py b/docarray/typing/tensor/embedding/__init__.py index aabe4272107..42814851999 100644 --- a/docarray/typing/tensor/embedding/__init__.py +++ b/docarray/typing/tensor/embedding/__init__.py @@ -2,14 +2,15 @@ from docarray.typing.tensor.embedding.embedding import AnyEmbedding from docarray.typing.tensor.embedding.ndarray import NdArrayEmbedding + +__all__ = ['NdArrayEmbedding', 'AnyEmbedding'] + from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from docarray.typing.tensor.embedding.tensorflow import TensorFlowEmbedding # noqa from docarray.typing.tensor.embedding.torch import TorchEmbedding # noqa -__all__ = ['NdArrayEmbedding', 'AnyEmbedding'] - def __getattr__(name: str): if name == 'TorchEmbedding': diff --git a/docarray/typing/tensor/image/__init__.py b/docarray/typing/tensor/image/__init__.py index 6dd56cc161c..1c199a5ff81 100644 --- a/docarray/typing/tensor/image/__init__.py +++ b/docarray/typing/tensor/image/__init__.py @@ -2,6 +2,9 @@ from docarray.typing.tensor.image.image_ndarray import ImageNdArray from docarray.typing.tensor.image.image_tensor import ImageTensor + +__all__ = ['ImageNdArray', 'ImageTensor'] + from docarray.utils._internal.misc import import_library if TYPE_CHECKING: @@ -10,8 +13,6 @@ ) from docarray.typing.tensor.image.image_torch_tensor import ImageTorchTensor # noqa -__all__ = ['ImageNdArray', 'ImageTensor'] - def __getattr__(name: str): if name == 'ImageTorchTensor': diff --git a/docarray/typing/tensor/video/__init__.py b/docarray/typing/tensor/video/__init__.py index f495a22cc13..8262e30f357 100644 --- a/docarray/typing/tensor/video/__init__.py +++ b/docarray/typing/tensor/video/__init__.py @@ -2,6 +2,9 @@ from docarray.typing.tensor.video.video_ndarray import VideoNdArray from docarray.typing.tensor.video.video_tensor import VideoTensor + +__all__ = ['VideoNdArray', 'VideoTensor'] + from docarray.utils._internal.misc import import_library if TYPE_CHECKING: @@ -10,8 +13,6 @@ ) from docarray.typing.tensor.video.video_torch_tensor import VideoTorchTensor # noqa -__all__ = ['VideoNdArray', 'VideoTensor'] - def __getattr__(name: str): if name == 'VideoTorchTensor': From f24042e091be4073d7bb64e9ba6d1c9c6e0e2e06 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 15:23:11 +0200 Subject: [PATCH 41/66] revert "fix: missed imports" This reverts commit 353f029207ca63a9c4e41b8391539b75557375b0. Signed-off-by: anna-charlotte --- docarray/base_doc/__init__.py | 2 +- docarray/base_doc/doc_response.py | 2 +- docarray/computation/tensorflow_backend.py | 2 +- docarray/computation/torch_backend.py | 2 +- docarray/documents/mesh/vertices_and_faces.py | 2 +- docarray/index/__init__.py | 2 +- docarray/proto/__init__.py | 2 +- docarray/store/__init__.py | 2 +- docarray/typing/bytes/audio_bytes.py | 2 +- docarray/typing/bytes/image_bytes.py | 2 +- docarray/typing/bytes/video_bytes.py | 2 +- docarray/typing/tensor/tensorflow_tensor.py | 2 +- docarray/typing/tensor/torch_tensor.py | 2 +- docarray/typing/url/url_3d/url_3d.py | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docarray/base_doc/__init__.py b/docarray/base_doc/__init__.py index 489ed0015a0..9e145403066 100644 --- a/docarray/base_doc/__init__.py +++ b/docarray/base_doc/__init__.py @@ -4,7 +4,7 @@ __all__ = ['AnyDoc', 'BaseDoc', 'BaseNode'] -from docarray.utils._internal.misc import import_library +from docarray.utils.misc import import_library def __getattr__(name: str): diff --git a/docarray/base_doc/doc_response.py b/docarray/base_doc/doc_response.py index 7d2b7b76ef6..f0a1686a4a8 100644 --- a/docarray/base_doc/doc_response.py +++ b/docarray/base_doc/doc_response.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING, Any -from docarray.utils._internal.misc import import_library +from docarray.utils.misc import import_library if TYPE_CHECKING: from fastapi.responses import JSONResponse diff --git a/docarray/computation/tensorflow_backend.py b/docarray/computation/tensorflow_backend.py index fc963cdb48b..41341e3066b 100644 --- a/docarray/computation/tensorflow_backend.py +++ b/docarray/computation/tensorflow_backend.py @@ -6,7 +6,7 @@ from docarray.computation import AbstractComputationalBackend from docarray.computation.abstract_numpy_based_backend import AbstractNumpyBasedBackend from docarray.typing import TensorFlowTensor -from docarray.utils._internal.misc import import_library +from docarray.utils.misc import import_library if TYPE_CHECKING: import tensorflow as tf # type: ignore diff --git a/docarray/computation/torch_backend.py b/docarray/computation/torch_backend.py index f64b993186a..0871fc2e689 100644 --- a/docarray/computation/torch_backend.py +++ b/docarray/computation/torch_backend.py @@ -3,7 +3,7 @@ import numpy as np from docarray.computation.abstract_comp_backend import AbstractComputationalBackend -from docarray.utils._internal.misc import import_library +from docarray.utils.misc import import_library if TYPE_CHECKING: import torch diff --git a/docarray/documents/mesh/vertices_and_faces.py b/docarray/documents/mesh/vertices_and_faces.py index cc3dc6da795..19697b1aa98 100644 --- a/docarray/documents/mesh/vertices_and_faces.py +++ b/docarray/documents/mesh/vertices_and_faces.py @@ -2,7 +2,7 @@ from docarray.base_doc import BaseDoc from docarray.typing.tensor.tensor import AnyTensor -from docarray.utils._internal.misc import import_library +from docarray.utils.misc import import_library T = TypeVar('T', bound='VerticesAndFaces') diff --git a/docarray/index/__init__.py b/docarray/index/__init__.py index 55a3e8dd318..1f6f8af8aca 100644 --- a/docarray/index/__init__.py +++ b/docarray/index/__init__.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING -from docarray.utils._internal.misc import import_library +from docarray.utils.misc import import_library __all__ = [] diff --git a/docarray/proto/__init__.py b/docarray/proto/__init__.py index 163809b7ef0..bca72030b79 100644 --- a/docarray/proto/__init__.py +++ b/docarray/proto/__init__.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING -from docarray.utils._internal.misc import import_library +from docarray.utils.misc import import_library if TYPE_CHECKING: from google.protobuf import __version__ as __pb__version__ diff --git a/docarray/store/__init__.py b/docarray/store/__init__.py index c1174042e41..1456c9986ab 100644 --- a/docarray/store/__init__.py +++ b/docarray/store/__init__.py @@ -5,7 +5,7 @@ __all__ = ['FileDocStore'] -from docarray.utils._internal.misc import import_library +from docarray.utils.misc import import_library if TYPE_CHECKING: from docarray.store.jac import JACDocStore # noqa: F401 diff --git a/docarray/typing/bytes/audio_bytes.py b/docarray/typing/bytes/audio_bytes.py index 643c753067c..aae0a628d35 100644 --- a/docarray/typing/bytes/audio_bytes.py +++ b/docarray/typing/bytes/audio_bytes.py @@ -7,7 +7,7 @@ from docarray.typing.abstract_type import AbstractType from docarray.typing.proto_register import _register_proto -from docarray.utils._internal.misc import import_library +from docarray.utils.misc import import_library if TYPE_CHECKING: from pydantic.fields import BaseConfig, ModelField diff --git a/docarray/typing/bytes/image_bytes.py b/docarray/typing/bytes/image_bytes.py index c7d2bf8401d..d21dbe8af9e 100644 --- a/docarray/typing/bytes/image_bytes.py +++ b/docarray/typing/bytes/image_bytes.py @@ -7,7 +7,7 @@ from docarray.typing.abstract_type import AbstractType from docarray.typing.proto_register import _register_proto -from docarray.utils._internal.misc import import_library +from docarray.utils.misc import import_library if TYPE_CHECKING: from pydantic.fields import BaseConfig, ModelField diff --git a/docarray/typing/bytes/video_bytes.py b/docarray/typing/bytes/video_bytes.py index 31b6cdc97a4..079924b4a87 100644 --- a/docarray/typing/bytes/video_bytes.py +++ b/docarray/typing/bytes/video_bytes.py @@ -8,7 +8,7 @@ from docarray.typing import AudioNdArray, NdArray, VideoNdArray from docarray.typing.abstract_type import AbstractType from docarray.typing.proto_register import _register_proto -from docarray.utils._internal.misc import import_library +from docarray.utils.misc import import_library if TYPE_CHECKING: from pydantic.fields import BaseConfig, ModelField diff --git a/docarray/typing/tensor/tensorflow_tensor.py b/docarray/typing/tensor/tensorflow_tensor.py index f0d79aa0b30..2329a8c689a 100644 --- a/docarray/typing/tensor/tensorflow_tensor.py +++ b/docarray/typing/tensor/tensorflow_tensor.py @@ -5,7 +5,7 @@ from docarray.base_doc.base_node import BaseNode from docarray.typing.proto_register import _register_proto from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._internal.misc import import_library +from docarray.utils.misc import import_library if TYPE_CHECKING: import tensorflow as tf # type: ignore diff --git a/docarray/typing/tensor/torch_tensor.py b/docarray/typing/tensor/torch_tensor.py index e2ece8c84dd..eb62cff2895 100644 --- a/docarray/typing/tensor/torch_tensor.py +++ b/docarray/typing/tensor/torch_tensor.py @@ -6,7 +6,7 @@ from docarray.base_doc.base_node import BaseNode from docarray.typing.proto_register import _register_proto from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils._internal.misc import import_library +from docarray.utils.misc import import_library if TYPE_CHECKING: import torch diff --git a/docarray/typing/url/url_3d/url_3d.py b/docarray/typing/url/url_3d/url_3d.py index b6d77b60829..b4349251131 100644 --- a/docarray/typing/url/url_3d/url_3d.py +++ b/docarray/typing/url/url_3d/url_3d.py @@ -3,7 +3,7 @@ from docarray.typing.proto_register import _register_proto from docarray.typing.url.any_url import AnyUrl -from docarray.utils._internal.misc import import_library +from docarray.utils.misc import import_library if TYPE_CHECKING: import trimesh From 08e3d46966b60d2282798db58e231a14501f31fa Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 15:41:49 +0200 Subject: [PATCH 42/66] fix: missed imports Signed-off-by: anna-charlotte --- docarray/base_doc/__init__.py | 2 +- docarray/base_doc/doc_response.py | 2 +- docarray/computation/tensorflow_backend.py | 2 +- docarray/computation/torch_backend.py | 2 +- docarray/documents/mesh/vertices_and_faces.py | 2 +- docarray/index/__init__.py | 2 +- docarray/proto/__init__.py | 2 +- docarray/store/__init__.py | 2 +- docarray/typing/bytes/audio_bytes.py | 2 +- docarray/typing/bytes/image_bytes.py | 2 +- docarray/typing/bytes/video_bytes.py | 2 +- docarray/typing/tensor/tensorflow_tensor.py | 2 +- docarray/typing/tensor/torch_tensor.py | 2 +- docarray/typing/url/url_3d/url_3d.py | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/docarray/base_doc/__init__.py b/docarray/base_doc/__init__.py index 9e145403066..489ed0015a0 100644 --- a/docarray/base_doc/__init__.py +++ b/docarray/base_doc/__init__.py @@ -4,7 +4,7 @@ __all__ = ['AnyDoc', 'BaseDoc', 'BaseNode'] -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library def __getattr__(name: str): diff --git a/docarray/base_doc/doc_response.py b/docarray/base_doc/doc_response.py index f0a1686a4a8..7d2b7b76ef6 100644 --- a/docarray/base_doc/doc_response.py +++ b/docarray/base_doc/doc_response.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING, Any -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from fastapi.responses import JSONResponse diff --git a/docarray/computation/tensorflow_backend.py b/docarray/computation/tensorflow_backend.py index 41341e3066b..fc963cdb48b 100644 --- a/docarray/computation/tensorflow_backend.py +++ b/docarray/computation/tensorflow_backend.py @@ -6,7 +6,7 @@ from docarray.computation import AbstractComputationalBackend from docarray.computation.abstract_numpy_based_backend import AbstractNumpyBasedBackend from docarray.typing import TensorFlowTensor -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: import tensorflow as tf # type: ignore diff --git a/docarray/computation/torch_backend.py b/docarray/computation/torch_backend.py index 0871fc2e689..f64b993186a 100644 --- a/docarray/computation/torch_backend.py +++ b/docarray/computation/torch_backend.py @@ -3,7 +3,7 @@ import numpy as np from docarray.computation.abstract_comp_backend import AbstractComputationalBackend -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: import torch diff --git a/docarray/documents/mesh/vertices_and_faces.py b/docarray/documents/mesh/vertices_and_faces.py index 19697b1aa98..cc3dc6da795 100644 --- a/docarray/documents/mesh/vertices_and_faces.py +++ b/docarray/documents/mesh/vertices_and_faces.py @@ -2,7 +2,7 @@ from docarray.base_doc import BaseDoc from docarray.typing.tensor.tensor import AnyTensor -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library T = TypeVar('T', bound='VerticesAndFaces') diff --git a/docarray/index/__init__.py b/docarray/index/__init__.py index 1f6f8af8aca..55a3e8dd318 100644 --- a/docarray/index/__init__.py +++ b/docarray/index/__init__.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library __all__ = [] diff --git a/docarray/proto/__init__.py b/docarray/proto/__init__.py index bca72030b79..163809b7ef0 100644 --- a/docarray/proto/__init__.py +++ b/docarray/proto/__init__.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from google.protobuf import __version__ as __pb__version__ diff --git a/docarray/store/__init__.py b/docarray/store/__init__.py index 1456c9986ab..c1174042e41 100644 --- a/docarray/store/__init__.py +++ b/docarray/store/__init__.py @@ -5,7 +5,7 @@ __all__ = ['FileDocStore'] -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from docarray.store.jac import JACDocStore # noqa: F401 diff --git a/docarray/typing/bytes/audio_bytes.py b/docarray/typing/bytes/audio_bytes.py index aae0a628d35..643c753067c 100644 --- a/docarray/typing/bytes/audio_bytes.py +++ b/docarray/typing/bytes/audio_bytes.py @@ -7,7 +7,7 @@ from docarray.typing.abstract_type import AbstractType from docarray.typing.proto_register import _register_proto -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from pydantic.fields import BaseConfig, ModelField diff --git a/docarray/typing/bytes/image_bytes.py b/docarray/typing/bytes/image_bytes.py index d21dbe8af9e..c7d2bf8401d 100644 --- a/docarray/typing/bytes/image_bytes.py +++ b/docarray/typing/bytes/image_bytes.py @@ -7,7 +7,7 @@ from docarray.typing.abstract_type import AbstractType from docarray.typing.proto_register import _register_proto -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from pydantic.fields import BaseConfig, ModelField diff --git a/docarray/typing/bytes/video_bytes.py b/docarray/typing/bytes/video_bytes.py index 079924b4a87..31b6cdc97a4 100644 --- a/docarray/typing/bytes/video_bytes.py +++ b/docarray/typing/bytes/video_bytes.py @@ -8,7 +8,7 @@ from docarray.typing import AudioNdArray, NdArray, VideoNdArray from docarray.typing.abstract_type import AbstractType from docarray.typing.proto_register import _register_proto -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from pydantic.fields import BaseConfig, ModelField diff --git a/docarray/typing/tensor/tensorflow_tensor.py b/docarray/typing/tensor/tensorflow_tensor.py index 2329a8c689a..f0d79aa0b30 100644 --- a/docarray/typing/tensor/tensorflow_tensor.py +++ b/docarray/typing/tensor/tensorflow_tensor.py @@ -5,7 +5,7 @@ from docarray.base_doc.base_node import BaseNode from docarray.typing.proto_register import _register_proto from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: import tensorflow as tf # type: ignore diff --git a/docarray/typing/tensor/torch_tensor.py b/docarray/typing/tensor/torch_tensor.py index eb62cff2895..e2ece8c84dd 100644 --- a/docarray/typing/tensor/torch_tensor.py +++ b/docarray/typing/tensor/torch_tensor.py @@ -6,7 +6,7 @@ from docarray.base_doc.base_node import BaseNode from docarray.typing.proto_register import _register_proto from docarray.typing.tensor.abstract_tensor import AbstractTensor -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: import torch diff --git a/docarray/typing/url/url_3d/url_3d.py b/docarray/typing/url/url_3d/url_3d.py index b4349251131..b6d77b60829 100644 --- a/docarray/typing/url/url_3d/url_3d.py +++ b/docarray/typing/url/url_3d/url_3d.py @@ -3,7 +3,7 @@ from docarray.typing.proto_register import _register_proto from docarray.typing.url.any_url import AnyUrl -from docarray.utils.misc import import_library +from docarray.utils._internal.misc import import_library if TYPE_CHECKING: import trimesh From 1fe5a1e4b0bbbef1f5d20e8a2cb8123383c967ca Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Tue, 28 Mar 2023 17:09:54 +0200 Subject: [PATCH 43/66] wip Signed-off-by: anna-charlotte --- docarray/typing/__init__.py | 168 ++++++++++++++++---------- tests/documentation/test_docstring.py | 3 + 2 files changed, 110 insertions(+), 61 deletions(-) diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index 1db60181415..f91534c426e 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -33,7 +33,6 @@ 'AnyUrl', 'ID', 'AnyTensor', - 'NdArrayEmbedding', 'ImageBytes', 'ImageTensor', 'ImageNdArray', @@ -52,64 +51,111 @@ from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa: F401 from docarray.typing.tensor.video import VideoTorchTensor # noqa: F401 +from docarray.utils._internal.misc import is_tf_available, is_torch_available -torch_tensors = [ - 'AudioTorchTensor', - 'TorchEmbedding', - 'TorchTensor', - 'VideoTorchTensor', - 'ImageTorchTensor', -] - -tf_tensors = [ - 'TensorFlowTensor', - 'TensorFlowEmbedding', - 'AudioTensorFlowTensor', - 'ImageTensorFlowTensor', - 'VideoTensorFlowTensor', -] - - -def __getattr__(name: str): - if name in torch_tensors: - import_library('torch', raise_error=True) - - from docarray.typing.tensor import TorchEmbedding, TorchTensor # noqa: F811 - from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F811 - from docarray.typing.tensor.image import ImageTorchTensor # noqa: F811 - from docarray.typing.tensor.video import VideoTorchTensor # noqa: F811 - - __all__.extend(torch_tensors) - - if name == 'TorchTensor': - return TorchTensor - elif name == 'TorchEmbedding': - return TorchEmbedding - elif name == 'AudioTorchTensor': - return AudioTorchTensor - elif name == 'ImageTorchTensor': - return ImageTorchTensor - elif name == 'VideoTorchTensor': - return VideoTorchTensor - - elif name in tf_tensors: - import_library('tensorflow', raise_error=True) - - from docarray.typing.tensor import TensorFlowTensor # noqa - from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa - from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa - from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa - from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa - - __all__.extend(torch_tensors) - - if name == 'TensorFlowTensor': - return TensorFlowTensor - elif name == 'TensorFlowEmbedding': - return TensorFlowEmbedding - elif name == 'AudioTensorFlowTensor': - return AudioTensorFlowTensor - elif name == 'ImageTensorFlowTensor': - return ImageTensorFlowTensor - elif name == 'VideoTensorFlowTensor': - return VideoTensorFlowTensor +torch_available = is_torch_available() +if torch_available: + from docarray.typing.tensor import TorchEmbedding, TorchTensor # noqa: F401 + from docarray.typing.tensor.audio.audio_torch_tensor import AudioTorchTensor # noqa + from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 + from docarray.typing.tensor.video.video_torch_tensor import VideoTorchTensor # noqa + + __all__.extend( + [ + 'AudioTorchTensor', + 'TorchEmbedding', + 'TorchTensor', + 'VideoTorchTensor', + 'ImageTorchTensor', + ] + ) + +tf_available = is_tf_available() +if tf_available: + from docarray.typing.tensor import TensorFlowTensor # noqa: F401 + from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 + from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa + + __all__.extend( + [ + 'TensorFlowTensor', + 'TensorFlowEmbedding', + 'AudioTensorFlowTensor', + 'ImageTensorFlowTensor', + 'VideoTensorFlowTensor', + ] + ) + +# +# def __getattr__(name: str): +# +# torch_tensors = [ +# 'AudioTorchTensor', +# 'TorchEmbedding', +# 'TorchTensor', +# 'VideoTorchTensor', +# 'ImageTorchTensor', +# ] +# +# tf_tensors = [ +# 'TensorFlowTensor', +# 'TensorFlowEmbedding', +# 'AudioTensorFlowTensor', +# 'ImageTensorFlowTensor', +# 'VideoTensorFlowTensor', +# ] +# +# if name in torch_tensors: +# import_library('torch', raise_error=True) +# +# from docarray.typing.tensor import TorchEmbedding, TorchTensor # noqa: F811 +# from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F811 +# from docarray.typing.tensor.image import ImageTorchTensor # noqa: F811 +# from docarray.typing.tensor.video import VideoTorchTensor # noqa: F811 +# +# # __all__.extend(torch_tensors) +# +# if name == 'TorchTensor': +# __all__.append('TorchTensor') +# return TorchTensor +# elif name == 'TorchEmbedding': +# __all__.append('TorchEmbedding') +# return TorchEmbedding +# elif name == 'AudioTorchTensor': +# __all__.append('AudioTorchTensor') +# return AudioTorchTensor +# elif name == 'ImageTorchTensor': +# __all__.append('ImageTorchTensor') +# return ImageTorchTensor +# elif name == 'VideoTorchTensor': +# __all__.append('VideoTorchTensor') +# return VideoTorchTensor +# +# elif name in tf_tensors: +# import_library('tensorflow', raise_error=True) +# +# from docarray.typing.tensor import TensorFlowTensor # noqa +# from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa +# from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa +# from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa +# from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa +# +# # __all__.extend(torch_tensors) +# +# if name == 'TensorFlowTensor': +# __all__.append('TensorFlowTensor') +# return TensorFlowTensor +# elif name == 'TensorFlowEmbedding': +# __all__.append('TensorFlowEmbedding') +# return TensorFlowEmbedding +# elif name == 'AudioTensorFlowTensor': +# __all__.append('AudioTensorFlowTensor') +# return AudioTensorFlowTensor +# elif name == 'ImageTensorFlowTensor': +# __all__.append('ImageTensorFlowTensor') +# return ImageTensorFlowTensor +# elif name == 'VideoTensorFlowTensor': +# __all__.append('VideoTensorFlowTensor') +# return VideoTensorFlowTensor diff --git a/tests/documentation/test_docstring.py b/tests/documentation/test_docstring.py index c6a8e3d9a40..3d3cafe2311 100644 --- a/tests/documentation/test_docstring.py +++ b/tests/documentation/test_docstring.py @@ -42,6 +42,9 @@ def get_obj_to_check(lib): for obj in obj_to_check: members.extend(get_codeblock_members(obj)) +print(f"len(members) = {len(members)}") +print(f"len(set(members)) = {len(set(members))}") + @pytest.mark.parametrize("obj", members, ids=lambda d: d.__qualname__) def test_member(obj): From cf19fa5b80b44855add82079df60a7351174571c Mon Sep 17 00:00:00 2001 From: samsja <55492238+samsja@users.noreply.github.com> Date: Tue, 28 Mar 2023 16:43:16 +0200 Subject: [PATCH 44/66] fix: rename DocArrayProto to DocumentArrayProto (#1297) Signed-off-by: samsja Signed-off-by: anna-charlotte --- docarray/array/abstract_array.py | 6 +-- docarray/array/array/array.py | 4 +- docarray/array/array/io.py | 14 +++--- docarray/array/stacked/array_stacked.py | 4 +- docarray/proto/__init__.py | 8 ++-- docarray/proto/docarray.proto | 6 +-- docarray/proto/pb/docarray_pb2.py | 52 ++++++++++---------- docarray/proto/pb2/docarray_pb2.py | 64 ++++++++++++------------- 8 files changed, 79 insertions(+), 79 deletions(-) diff --git a/docarray/array/abstract_array.py b/docarray/array/abstract_array.py index 0669a927451..f3a94b5b6fa 100644 --- a/docarray/array/abstract_array.py +++ b/docarray/array/abstract_array.py @@ -25,7 +25,7 @@ from docarray.utils._internal._typing import change_cls_name if TYPE_CHECKING: - from docarray.proto import DocArrayProto, NodeProto + from docarray.proto import DocumentArrayProto, NodeProto from docarray.typing.tensor.abstract_tensor import AbstractTensor T = TypeVar('T', bound='AnyDocArray') @@ -130,12 +130,12 @@ def _set_data_column( @classmethod @abstractmethod - def from_protobuf(cls: Type[T], pb_msg: 'DocArrayProto') -> T: + def from_protobuf(cls: Type[T], pb_msg: 'DocumentArrayProto') -> T: """create a Document from a protobuf message""" ... @abstractmethod - def to_protobuf(self) -> 'DocArrayProto': + def to_protobuf(self) -> 'DocumentArrayProto': """Convert DocArray into a Protobuf message""" ... diff --git a/docarray/array/array/array.py b/docarray/array/array/array.py index 1dac16e2eed..73f1a25a17b 100644 --- a/docarray/array/array/array.py +++ b/docarray/array/array/array.py @@ -32,7 +32,7 @@ from pydantic.fields import ModelField from docarray.array.stacked.array_stacked import DocArrayStacked - from docarray.proto import DocArrayProto + from docarray.proto import DocumentArrayProto from docarray.typing import TorchTensor from docarray.typing.tensor.abstract_tensor import AbstractTensor @@ -285,7 +285,7 @@ def traverse_flat( return flattened @classmethod - def from_protobuf(cls: Type[T], pb_msg: 'DocArrayProto') -> T: + def from_protobuf(cls: Type[T], pb_msg: 'DocumentArrayProto') -> T: """create a Document from a protobuf message :param pb_msg: The protobuf message from where to construct the DocArray """ diff --git a/docarray/array/array/io.py b/docarray/array/array/io.py index 2b8d6a72ff3..251afc65a4a 100644 --- a/docarray/array/array/io.py +++ b/docarray/array/array/io.py @@ -38,7 +38,7 @@ import pandas as pd from docarray import DocArray - from docarray.proto import DocArrayProto + from docarray.proto import DocumentArrayProto T = TypeVar('T', bound='IOMixinArray') @@ -109,7 +109,7 @@ def __init__( ... @classmethod - def from_protobuf(cls: Type[T], pb_msg: 'DocArrayProto') -> T: + def from_protobuf(cls: Type[T], pb_msg: 'DocumentArrayProto') -> T: """create a Document from a protobuf message :param pb_msg: The protobuf message from where to construct the DocArray """ @@ -117,11 +117,11 @@ def from_protobuf(cls: Type[T], pb_msg: 'DocArrayProto') -> T: cls.document_type.from_protobuf(doc_proto) for doc_proto in pb_msg.docs ) - def to_protobuf(self) -> 'DocArrayProto': + def to_protobuf(self) -> 'DocumentArrayProto': """Convert DocArray into a Protobuf message""" - from docarray.proto import DocArrayProto + from docarray.proto import DocumentArrayProto - da_proto = DocArrayProto() + da_proto = DocumentArrayProto() for doc in self: da_proto.docs.append(doc.to_protobuf()) @@ -555,9 +555,9 @@ def _load_binary_all( compress = None if protocol is not None and protocol == 'protobuf-array': - from docarray.proto import DocArrayProto + from docarray.proto import DocumentArrayProto - dap = DocArrayProto() + dap = DocumentArrayProto() dap.ParseFromString(d) return cls.from_protobuf(dap) diff --git a/docarray/array/stacked/array_stacked.py b/docarray/array/stacked/array_stacked.py index 1445b97379e..bc6ff8965a5 100644 --- a/docarray/array/stacked/array_stacked.py +++ b/docarray/array/stacked/array_stacked.py @@ -436,14 +436,14 @@ def from_protobuf(cls: Type[T], pb_msg: 'DocArrayStackedProto') -> T: def to_protobuf(self) -> 'DocArrayStackedProto': """Convert DocArray into a Protobuf message""" from docarray.proto import ( - DocArrayProto, DocArrayStackedProto, + DocumentArrayProto, ListOfAnyProto, ListOfDocArrayProto, NdArrayProto, ) - da_proto = DocArrayProto() + da_proto = DocumentArrayProto() for doc in self: da_proto.docs.append(doc.to_protobuf()) diff --git a/docarray/proto/__init__.py b/docarray/proto/__init__.py index 163809b7ef0..1b04df23fe6 100644 --- a/docarray/proto/__init__.py +++ b/docarray/proto/__init__.py @@ -12,8 +12,8 @@ if __pb__version__.startswith('4'): from docarray.proto.pb.docarray_pb2 import ( DictOfAnyProto, - DocArrayProto, DocArrayStackedProto, + DocumentArrayProto, DocumentProto, ListOfAnyProto, ListOfDocArrayProto, @@ -23,8 +23,8 @@ else: from docarray.proto.pb2.docarray_pb2 import ( DictOfAnyProto, - DocArrayProto, DocArrayStackedProto, + DocumentArrayProto, DocumentProto, ListOfAnyProto, ListOfDocArrayProto, @@ -33,12 +33,12 @@ ) __all__ = [ - 'DocArrayProto', + 'DocumentArrayProto', 'DocumentProto', 'NdArrayProto', 'NodeProto', 'DocArrayStackedProto', - 'DocArrayProto', + 'DocumentArrayProto', 'ListOfDocArrayProto', 'ListOfAnyProto', 'DictOfAnyProto', diff --git a/docarray/proto/docarray.proto b/docarray/proto/docarray.proto index 85c6a882f0b..2b1d557da52 100644 --- a/docarray/proto/docarray.proto +++ b/docarray/proto/docarray.proto @@ -55,7 +55,7 @@ message NodeProto { // a sub Document DocumentProto document = 7; // a sub DocArray - DocArrayProto document_array = 8; + DocumentArrayProto document_array = 8; //any list ListOfAnyProto list = 9; //any set @@ -91,13 +91,13 @@ message ListOfAnyProto { repeated NodeProto data = 1; } -message DocArrayProto { +message DocumentArrayProto { repeated DocumentProto docs = 1; // a list of Documents } message ListOfDocArrayProto { - repeated DocArrayProto data = 1; + repeated DocumentArrayProto data = 1; } message DocArrayStackedProto{ diff --git a/docarray/proto/pb/docarray_pb2.py b/docarray/proto/pb/docarray_pb2.py index b3b0af6d3ac..f6eeaf8fdb9 100644 --- a/docarray/proto/pb/docarray_pb2.py +++ b/docarray/proto/pb/docarray_pb2.py @@ -16,7 +16,7 @@ from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0e\x64ocarray.proto\x12\x08\x64ocarray\x1a\x1cgoogle/protobuf/struct.proto\"A\n\x11\x44\x65nseNdArrayProto\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\x0c\x12\r\n\x05shape\x18\x02 \x03(\r\x12\r\n\x05\x64type\x18\x03 \x01(\t\"g\n\x0cNdArrayProto\x12*\n\x05\x64\x65nse\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12+\n\nparameters\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\"Z\n\x0cKeyValuePair\x12#\n\x03key\x18\x01 \x01(\x0b\x32\x16.google.protobuf.Value\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value\";\n\x10GenericDictValue\x12\'\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x16.docarray.KeyValuePair\"\xc1\x03\n\tNodeProto\x12\x0e\n\x04text\x18\x01 \x01(\tH\x00\x12\x11\n\x07integer\x18\x02 \x01(\x05H\x00\x12\x0f\n\x05\x66loat\x18\x03 \x01(\x01H\x00\x12\x11\n\x07\x62oolean\x18\x04 \x01(\x08H\x00\x12\x0e\n\x04\x62lob\x18\x05 \x01(\x0cH\x00\x12)\n\x07ndarray\x18\x06 \x01(\x0b\x32\x16.docarray.NdArrayProtoH\x00\x12+\n\x08\x64ocument\x18\x07 \x01(\x0b\x32\x17.docarray.DocumentProtoH\x00\x12\x31\n\x0e\x64ocument_array\x18\x08 \x01(\x0b\x32\x17.docarray.DocArrayProtoH\x00\x12(\n\x04list\x18\t \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12\'\n\x03set\x18\n \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12)\n\x05tuple\x18\x0b \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12(\n\x04\x64ict\x18\x0c \x01(\x0b\x32\x18.docarray.DictOfAnyProtoH\x00\x12\x0e\n\x04type\x18\r \x01(\tH\x01\x42\t\n\x07\x63ontentB\x0f\n\rdocarray_type\"\x82\x01\n\rDocumentProto\x12/\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32!.docarray.DocumentProto.DataEntry\x1a@\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.docarray.NodeProto:\x02\x38\x01\"\x84\x01\n\x0e\x44ictOfAnyProto\x12\x30\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\".docarray.DictOfAnyProto.DataEntry\x1a@\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.docarray.NodeProto:\x02\x38\x01\"3\n\x0eListOfAnyProto\x12!\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x13.docarray.NodeProto\"6\n\rDocArrayProto\x12%\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x17.docarray.DocumentProto\"<\n\x13ListOfDocArrayProto\x12%\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x17.docarray.DocArrayProto\"\xed\x04\n\x14\x44ocArrayStackedProto\x12I\n\x0etensor_columns\x18\x01 \x03(\x0b\x32\x31.docarray.DocArrayStackedProto.TensorColumnsEntry\x12\x43\n\x0b\x64oc_columns\x18\x02 \x03(\x0b\x32..docarray.DocArrayStackedProto.DocColumnsEntry\x12\x41\n\nda_columns\x18\x03 \x03(\x0b\x32-.docarray.DocArrayStackedProto.DaColumnsEntry\x12\x43\n\x0b\x61ny_columns\x18\x04 \x03(\x0b\x32..docarray.DocArrayStackedProto.AnyColumnsEntry\x1aL\n\x12TensorColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.docarray.NdArrayProto:\x02\x38\x01\x1aQ\n\x0f\x44ocColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12-\n\x05value\x18\x02 \x01(\x0b\x32\x1e.docarray.DocArrayStackedProto:\x02\x38\x01\x1aO\n\x0e\x44\x61\x43olumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.docarray.ListOfDocArrayProto:\x02\x38\x01\x1aK\n\x0f\x41nyColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\'\n\x05value\x18\x02 \x01(\x0b\x32\x18.docarray.ListOfAnyProto:\x02\x38\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0e\x64ocarray.proto\x12\x08\x64ocarray\x1a\x1cgoogle/protobuf/struct.proto\"A\n\x11\x44\x65nseNdArrayProto\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\x0c\x12\r\n\x05shape\x18\x02 \x03(\r\x12\r\n\x05\x64type\x18\x03 \x01(\t\"g\n\x0cNdArrayProto\x12*\n\x05\x64\x65nse\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12+\n\nparameters\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\"Z\n\x0cKeyValuePair\x12#\n\x03key\x18\x01 \x01(\x0b\x32\x16.google.protobuf.Value\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value\";\n\x10GenericDictValue\x12\'\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x16.docarray.KeyValuePair\"\xc6\x03\n\tNodeProto\x12\x0e\n\x04text\x18\x01 \x01(\tH\x00\x12\x11\n\x07integer\x18\x02 \x01(\x05H\x00\x12\x0f\n\x05\x66loat\x18\x03 \x01(\x01H\x00\x12\x11\n\x07\x62oolean\x18\x04 \x01(\x08H\x00\x12\x0e\n\x04\x62lob\x18\x05 \x01(\x0cH\x00\x12)\n\x07ndarray\x18\x06 \x01(\x0b\x32\x16.docarray.NdArrayProtoH\x00\x12+\n\x08\x64ocument\x18\x07 \x01(\x0b\x32\x17.docarray.DocumentProtoH\x00\x12\x36\n\x0e\x64ocument_array\x18\x08 \x01(\x0b\x32\x1c.docarray.DocumentArrayProtoH\x00\x12(\n\x04list\x18\t \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12\'\n\x03set\x18\n \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12)\n\x05tuple\x18\x0b \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12(\n\x04\x64ict\x18\x0c \x01(\x0b\x32\x18.docarray.DictOfAnyProtoH\x00\x12\x0e\n\x04type\x18\r \x01(\tH\x01\x42\t\n\x07\x63ontentB\x0f\n\rdocarray_type\"\x82\x01\n\rDocumentProto\x12/\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32!.docarray.DocumentProto.DataEntry\x1a@\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.docarray.NodeProto:\x02\x38\x01\"\x84\x01\n\x0e\x44ictOfAnyProto\x12\x30\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\".docarray.DictOfAnyProto.DataEntry\x1a@\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.docarray.NodeProto:\x02\x38\x01\"3\n\x0eListOfAnyProto\x12!\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x13.docarray.NodeProto\";\n\x12\x44ocumentArrayProto\x12%\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x17.docarray.DocumentProto\"A\n\x13ListOfDocArrayProto\x12*\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x1c.docarray.DocumentArrayProto\"\xed\x04\n\x14\x44ocArrayStackedProto\x12I\n\x0etensor_columns\x18\x01 \x03(\x0b\x32\x31.docarray.DocArrayStackedProto.TensorColumnsEntry\x12\x43\n\x0b\x64oc_columns\x18\x02 \x03(\x0b\x32..docarray.DocArrayStackedProto.DocColumnsEntry\x12\x41\n\nda_columns\x18\x03 \x03(\x0b\x32-.docarray.DocArrayStackedProto.DaColumnsEntry\x12\x43\n\x0b\x61ny_columns\x18\x04 \x03(\x0b\x32..docarray.DocArrayStackedProto.AnyColumnsEntry\x1aL\n\x12TensorColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.docarray.NdArrayProto:\x02\x38\x01\x1aQ\n\x0f\x44ocColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12-\n\x05value\x18\x02 \x01(\x0b\x32\x1e.docarray.DocArrayStackedProto:\x02\x38\x01\x1aO\n\x0e\x44\x61\x43olumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.docarray.ListOfDocArrayProto:\x02\x38\x01\x1aK\n\x0f\x41nyColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\'\n\x05value\x18\x02 \x01(\x0b\x32\x18.docarray.ListOfAnyProto:\x02\x38\x01\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'docarray_pb2', globals()) @@ -44,29 +44,29 @@ _GENERICDICTVALUE._serialized_start=322 _GENERICDICTVALUE._serialized_end=381 _NODEPROTO._serialized_start=384 - _NODEPROTO._serialized_end=833 - _DOCUMENTPROTO._serialized_start=836 - _DOCUMENTPROTO._serialized_end=966 - _DOCUMENTPROTO_DATAENTRY._serialized_start=902 - _DOCUMENTPROTO_DATAENTRY._serialized_end=966 - _DICTOFANYPROTO._serialized_start=969 - _DICTOFANYPROTO._serialized_end=1101 - _DICTOFANYPROTO_DATAENTRY._serialized_start=902 - _DICTOFANYPROTO_DATAENTRY._serialized_end=966 - _LISTOFANYPROTO._serialized_start=1103 - _LISTOFANYPROTO._serialized_end=1154 - _DOCARRAYPROTO._serialized_start=1156 - _DOCARRAYPROTO._serialized_end=1210 - _LISTOFDOCARRAYPROTO._serialized_start=1212 - _LISTOFDOCARRAYPROTO._serialized_end=1272 - _DOCARRAYSTACKEDPROTO._serialized_start=1275 - _DOCARRAYSTACKEDPROTO._serialized_end=1896 - _DOCARRAYSTACKEDPROTO_TENSORCOLUMNSENTRY._serialized_start=1579 - _DOCARRAYSTACKEDPROTO_TENSORCOLUMNSENTRY._serialized_end=1655 - _DOCARRAYSTACKEDPROTO_DOCCOLUMNSENTRY._serialized_start=1657 - _DOCARRAYSTACKEDPROTO_DOCCOLUMNSENTRY._serialized_end=1738 - _DOCARRAYSTACKEDPROTO_DACOLUMNSENTRY._serialized_start=1740 - _DOCARRAYSTACKEDPROTO_DACOLUMNSENTRY._serialized_end=1819 - _DOCARRAYSTACKEDPROTO_ANYCOLUMNSENTRY._serialized_start=1821 - _DOCARRAYSTACKEDPROTO_ANYCOLUMNSENTRY._serialized_end=1896 + _NODEPROTO._serialized_end=838 + _DOCUMENTPROTO._serialized_start=841 + _DOCUMENTPROTO._serialized_end=971 + _DOCUMENTPROTO_DATAENTRY._serialized_start=907 + _DOCUMENTPROTO_DATAENTRY._serialized_end=971 + _DICTOFANYPROTO._serialized_start=974 + _DICTOFANYPROTO._serialized_end=1106 + _DICTOFANYPROTO_DATAENTRY._serialized_start=907 + _DICTOFANYPROTO_DATAENTRY._serialized_end=971 + _LISTOFANYPROTO._serialized_start=1108 + _LISTOFANYPROTO._serialized_end=1159 + _DOCUMENTARRAYPROTO._serialized_start=1161 + _DOCUMENTARRAYPROTO._serialized_end=1220 + _LISTOFDOCARRAYPROTO._serialized_start=1222 + _LISTOFDOCARRAYPROTO._serialized_end=1287 + _DOCARRAYSTACKEDPROTO._serialized_start=1290 + _DOCARRAYSTACKEDPROTO._serialized_end=1911 + _DOCARRAYSTACKEDPROTO_TENSORCOLUMNSENTRY._serialized_start=1594 + _DOCARRAYSTACKEDPROTO_TENSORCOLUMNSENTRY._serialized_end=1670 + _DOCARRAYSTACKEDPROTO_DOCCOLUMNSENTRY._serialized_start=1672 + _DOCARRAYSTACKEDPROTO_DOCCOLUMNSENTRY._serialized_end=1753 + _DOCARRAYSTACKEDPROTO_DACOLUMNSENTRY._serialized_start=1755 + _DOCARRAYSTACKEDPROTO_DACOLUMNSENTRY._serialized_end=1834 + _DOCARRAYSTACKEDPROTO_ANYCOLUMNSENTRY._serialized_start=1836 + _DOCARRAYSTACKEDPROTO_ANYCOLUMNSENTRY._serialized_end=1911 # @@protoc_insertion_point(module_scope) diff --git a/docarray/proto/pb2/docarray_pb2.py b/docarray/proto/pb2/docarray_pb2.py index f07dbd806b1..0ea41987658 100644 --- a/docarray/proto/pb2/docarray_pb2.py +++ b/docarray/proto/pb2/docarray_pb2.py @@ -15,7 +15,7 @@ from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x0e\x64ocarray.proto\x12\x08\x64ocarray\x1a\x1cgoogle/protobuf/struct.proto\"A\n\x11\x44\x65nseNdArrayProto\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\x0c\x12\r\n\x05shape\x18\x02 \x03(\r\x12\r\n\x05\x64type\x18\x03 \x01(\t\"g\n\x0cNdArrayProto\x12*\n\x05\x64\x65nse\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12+\n\nparameters\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\"Z\n\x0cKeyValuePair\x12#\n\x03key\x18\x01 \x01(\x0b\x32\x16.google.protobuf.Value\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value\";\n\x10GenericDictValue\x12\'\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x16.docarray.KeyValuePair\"\xc1\x03\n\tNodeProto\x12\x0e\n\x04text\x18\x01 \x01(\tH\x00\x12\x11\n\x07integer\x18\x02 \x01(\x05H\x00\x12\x0f\n\x05\x66loat\x18\x03 \x01(\x01H\x00\x12\x11\n\x07\x62oolean\x18\x04 \x01(\x08H\x00\x12\x0e\n\x04\x62lob\x18\x05 \x01(\x0cH\x00\x12)\n\x07ndarray\x18\x06 \x01(\x0b\x32\x16.docarray.NdArrayProtoH\x00\x12+\n\x08\x64ocument\x18\x07 \x01(\x0b\x32\x17.docarray.DocumentProtoH\x00\x12\x31\n\x0e\x64ocument_array\x18\x08 \x01(\x0b\x32\x17.docarray.DocArrayProtoH\x00\x12(\n\x04list\x18\t \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12\'\n\x03set\x18\n \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12)\n\x05tuple\x18\x0b \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12(\n\x04\x64ict\x18\x0c \x01(\x0b\x32\x18.docarray.DictOfAnyProtoH\x00\x12\x0e\n\x04type\x18\r \x01(\tH\x01\x42\t\n\x07\x63ontentB\x0f\n\rdocarray_type\"\x82\x01\n\rDocumentProto\x12/\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32!.docarray.DocumentProto.DataEntry\x1a@\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.docarray.NodeProto:\x02\x38\x01\"\x84\x01\n\x0e\x44ictOfAnyProto\x12\x30\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\".docarray.DictOfAnyProto.DataEntry\x1a@\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.docarray.NodeProto:\x02\x38\x01\"3\n\x0eListOfAnyProto\x12!\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x13.docarray.NodeProto\"6\n\rDocArrayProto\x12%\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x17.docarray.DocumentProto\"<\n\x13ListOfDocArrayProto\x12%\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x17.docarray.DocArrayProto\"\xed\x04\n\x14\x44ocArrayStackedProto\x12I\n\x0etensor_columns\x18\x01 \x03(\x0b\x32\x31.docarray.DocArrayStackedProto.TensorColumnsEntry\x12\x43\n\x0b\x64oc_columns\x18\x02 \x03(\x0b\x32..docarray.DocArrayStackedProto.DocColumnsEntry\x12\x41\n\nda_columns\x18\x03 \x03(\x0b\x32-.docarray.DocArrayStackedProto.DaColumnsEntry\x12\x43\n\x0b\x61ny_columns\x18\x04 \x03(\x0b\x32..docarray.DocArrayStackedProto.AnyColumnsEntry\x1aL\n\x12TensorColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.docarray.NdArrayProto:\x02\x38\x01\x1aQ\n\x0f\x44ocColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12-\n\x05value\x18\x02 \x01(\x0b\x32\x1e.docarray.DocArrayStackedProto:\x02\x38\x01\x1aO\n\x0e\x44\x61\x43olumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.docarray.ListOfDocArrayProto:\x02\x38\x01\x1aK\n\x0f\x41nyColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\'\n\x05value\x18\x02 \x01(\x0b\x32\x18.docarray.ListOfAnyProto:\x02\x38\x01\x62\x06proto3' + b'\n\x0e\x64ocarray.proto\x12\x08\x64ocarray\x1a\x1cgoogle/protobuf/struct.proto\"A\n\x11\x44\x65nseNdArrayProto\x12\x0e\n\x06\x62uffer\x18\x01 \x01(\x0c\x12\r\n\x05shape\x18\x02 \x03(\r\x12\r\n\x05\x64type\x18\x03 \x01(\t\"g\n\x0cNdArrayProto\x12*\n\x05\x64\x65nse\x18\x01 \x01(\x0b\x32\x1b.docarray.DenseNdArrayProto\x12+\n\nparameters\x18\x02 \x01(\x0b\x32\x17.google.protobuf.Struct\"Z\n\x0cKeyValuePair\x12#\n\x03key\x18\x01 \x01(\x0b\x32\x16.google.protobuf.Value\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.Value\";\n\x10GenericDictValue\x12\'\n\x07\x65ntries\x18\x01 \x03(\x0b\x32\x16.docarray.KeyValuePair\"\xc6\x03\n\tNodeProto\x12\x0e\n\x04text\x18\x01 \x01(\tH\x00\x12\x11\n\x07integer\x18\x02 \x01(\x05H\x00\x12\x0f\n\x05\x66loat\x18\x03 \x01(\x01H\x00\x12\x11\n\x07\x62oolean\x18\x04 \x01(\x08H\x00\x12\x0e\n\x04\x62lob\x18\x05 \x01(\x0cH\x00\x12)\n\x07ndarray\x18\x06 \x01(\x0b\x32\x16.docarray.NdArrayProtoH\x00\x12+\n\x08\x64ocument\x18\x07 \x01(\x0b\x32\x17.docarray.DocumentProtoH\x00\x12\x36\n\x0e\x64ocument_array\x18\x08 \x01(\x0b\x32\x1c.docarray.DocumentArrayProtoH\x00\x12(\n\x04list\x18\t \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12\'\n\x03set\x18\n \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12)\n\x05tuple\x18\x0b \x01(\x0b\x32\x18.docarray.ListOfAnyProtoH\x00\x12(\n\x04\x64ict\x18\x0c \x01(\x0b\x32\x18.docarray.DictOfAnyProtoH\x00\x12\x0e\n\x04type\x18\r \x01(\tH\x01\x42\t\n\x07\x63ontentB\x0f\n\rdocarray_type\"\x82\x01\n\rDocumentProto\x12/\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32!.docarray.DocumentProto.DataEntry\x1a@\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.docarray.NodeProto:\x02\x38\x01\"\x84\x01\n\x0e\x44ictOfAnyProto\x12\x30\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\".docarray.DictOfAnyProto.DataEntry\x1a@\n\tDataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.docarray.NodeProto:\x02\x38\x01\"3\n\x0eListOfAnyProto\x12!\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x13.docarray.NodeProto\";\n\x12\x44ocumentArrayProto\x12%\n\x04\x64ocs\x18\x01 \x03(\x0b\x32\x17.docarray.DocumentProto\"A\n\x13ListOfDocArrayProto\x12*\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32\x1c.docarray.DocumentArrayProto\"\xed\x04\n\x14\x44ocArrayStackedProto\x12I\n\x0etensor_columns\x18\x01 \x03(\x0b\x32\x31.docarray.DocArrayStackedProto.TensorColumnsEntry\x12\x43\n\x0b\x64oc_columns\x18\x02 \x03(\x0b\x32..docarray.DocArrayStackedProto.DocColumnsEntry\x12\x41\n\nda_columns\x18\x03 \x03(\x0b\x32-.docarray.DocArrayStackedProto.DaColumnsEntry\x12\x43\n\x0b\x61ny_columns\x18\x04 \x03(\x0b\x32..docarray.DocArrayStackedProto.AnyColumnsEntry\x1aL\n\x12TensorColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12%\n\x05value\x18\x02 \x01(\x0b\x32\x16.docarray.NdArrayProto:\x02\x38\x01\x1aQ\n\x0f\x44ocColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12-\n\x05value\x18\x02 \x01(\x0b\x32\x1e.docarray.DocArrayStackedProto:\x02\x38\x01\x1aO\n\x0e\x44\x61\x43olumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x1d.docarray.ListOfDocArrayProto:\x02\x38\x01\x1aK\n\x0f\x41nyColumnsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\'\n\x05value\x18\x02 \x01(\x0b\x32\x18.docarray.ListOfAnyProto:\x02\x38\x01\x62\x06proto3' ) @@ -29,7 +29,7 @@ _DICTOFANYPROTO = DESCRIPTOR.message_types_by_name['DictOfAnyProto'] _DICTOFANYPROTO_DATAENTRY = _DICTOFANYPROTO.nested_types_by_name['DataEntry'] _LISTOFANYPROTO = DESCRIPTOR.message_types_by_name['ListOfAnyProto'] -_DOCARRAYPROTO = DESCRIPTOR.message_types_by_name['DocArrayProto'] +_DOCUMENTARRAYPROTO = DESCRIPTOR.message_types_by_name['DocumentArrayProto'] _LISTOFDOCARRAYPROTO = DESCRIPTOR.message_types_by_name['ListOfDocArrayProto'] _DOCARRAYSTACKEDPROTO = DESCRIPTOR.message_types_by_name['DocArrayStackedProto'] _DOCARRAYSTACKEDPROTO_TENSORCOLUMNSENTRY = _DOCARRAYSTACKEDPROTO.nested_types_by_name[ @@ -152,16 +152,16 @@ ) _sym_db.RegisterMessage(ListOfAnyProto) -DocArrayProto = _reflection.GeneratedProtocolMessageType( - 'DocArrayProto', +DocumentArrayProto = _reflection.GeneratedProtocolMessageType( + 'DocumentArrayProto', (_message.Message,), { - 'DESCRIPTOR': _DOCARRAYPROTO, + 'DESCRIPTOR': _DOCUMENTARRAYPROTO, '__module__': 'docarray_pb2' - # @@protoc_insertion_point(class_scope:docarray.DocArrayProto) + # @@protoc_insertion_point(class_scope:docarray.DocumentArrayProto) }, ) -_sym_db.RegisterMessage(DocArrayProto) +_sym_db.RegisterMessage(DocumentArrayProto) ListOfDocArrayProto = _reflection.GeneratedProtocolMessageType( 'ListOfDocArrayProto', @@ -249,29 +249,29 @@ _GENERICDICTVALUE._serialized_start = 322 _GENERICDICTVALUE._serialized_end = 381 _NODEPROTO._serialized_start = 384 - _NODEPROTO._serialized_end = 833 - _DOCUMENTPROTO._serialized_start = 836 - _DOCUMENTPROTO._serialized_end = 966 - _DOCUMENTPROTO_DATAENTRY._serialized_start = 902 - _DOCUMENTPROTO_DATAENTRY._serialized_end = 966 - _DICTOFANYPROTO._serialized_start = 969 - _DICTOFANYPROTO._serialized_end = 1101 - _DICTOFANYPROTO_DATAENTRY._serialized_start = 902 - _DICTOFANYPROTO_DATAENTRY._serialized_end = 966 - _LISTOFANYPROTO._serialized_start = 1103 - _LISTOFANYPROTO._serialized_end = 1154 - _DOCARRAYPROTO._serialized_start = 1156 - _DOCARRAYPROTO._serialized_end = 1210 - _LISTOFDOCARRAYPROTO._serialized_start = 1212 - _LISTOFDOCARRAYPROTO._serialized_end = 1272 - _DOCARRAYSTACKEDPROTO._serialized_start = 1275 - _DOCARRAYSTACKEDPROTO._serialized_end = 1896 - _DOCARRAYSTACKEDPROTO_TENSORCOLUMNSENTRY._serialized_start = 1579 - _DOCARRAYSTACKEDPROTO_TENSORCOLUMNSENTRY._serialized_end = 1655 - _DOCARRAYSTACKEDPROTO_DOCCOLUMNSENTRY._serialized_start = 1657 - _DOCARRAYSTACKEDPROTO_DOCCOLUMNSENTRY._serialized_end = 1738 - _DOCARRAYSTACKEDPROTO_DACOLUMNSENTRY._serialized_start = 1740 - _DOCARRAYSTACKEDPROTO_DACOLUMNSENTRY._serialized_end = 1819 - _DOCARRAYSTACKEDPROTO_ANYCOLUMNSENTRY._serialized_start = 1821 - _DOCARRAYSTACKEDPROTO_ANYCOLUMNSENTRY._serialized_end = 1896 + _NODEPROTO._serialized_end = 838 + _DOCUMENTPROTO._serialized_start = 841 + _DOCUMENTPROTO._serialized_end = 971 + _DOCUMENTPROTO_DATAENTRY._serialized_start = 907 + _DOCUMENTPROTO_DATAENTRY._serialized_end = 971 + _DICTOFANYPROTO._serialized_start = 974 + _DICTOFANYPROTO._serialized_end = 1106 + _DICTOFANYPROTO_DATAENTRY._serialized_start = 907 + _DICTOFANYPROTO_DATAENTRY._serialized_end = 971 + _LISTOFANYPROTO._serialized_start = 1108 + _LISTOFANYPROTO._serialized_end = 1159 + _DOCUMENTARRAYPROTO._serialized_start = 1161 + _DOCUMENTARRAYPROTO._serialized_end = 1220 + _LISTOFDOCARRAYPROTO._serialized_start = 1222 + _LISTOFDOCARRAYPROTO._serialized_end = 1287 + _DOCARRAYSTACKEDPROTO._serialized_start = 1290 + _DOCARRAYSTACKEDPROTO._serialized_end = 1911 + _DOCARRAYSTACKEDPROTO_TENSORCOLUMNSENTRY._serialized_start = 1594 + _DOCARRAYSTACKEDPROTO_TENSORCOLUMNSENTRY._serialized_end = 1670 + _DOCARRAYSTACKEDPROTO_DOCCOLUMNSENTRY._serialized_start = 1672 + _DOCARRAYSTACKEDPROTO_DOCCOLUMNSENTRY._serialized_end = 1753 + _DOCARRAYSTACKEDPROTO_DACOLUMNSENTRY._serialized_start = 1755 + _DOCARRAYSTACKEDPROTO_DACOLUMNSENTRY._serialized_end = 1834 + _DOCARRAYSTACKEDPROTO_ANYCOLUMNSENTRY._serialized_start = 1836 + _DOCARRAYSTACKEDPROTO_ANYCOLUMNSENTRY._serialized_end = 1911 # @@protoc_insertion_point(module_scope) From 5f237e7e52bc9a6f0051a40e412532563bb34f67 Mon Sep 17 00:00:00 2001 From: samsja <55492238+samsja@users.noreply.github.com> Date: Tue, 28 Mar 2023 17:03:08 +0200 Subject: [PATCH 45/66] fix: docstring polish typing (#1299) * wip Signed-off-by: samsja * fix: cleanup namespace utils Signed-off-by: samsja * feat: add docstring test Signed-off-by: samsja * fix: fix video url docstring Signed-off-by: samsja * fix: fix text url Signed-off-by: samsja * fix: fix image url Signed-off-by: samsja * fix: fic audio url Signed-off-by: samsja * fix: mesh 3d url Signed-off-by: samsja * fix: mesh 3d url Signed-off-by: samsja * fix: remove useless data Signed-off-by: samsja * fix: fix docstring ndarray and torch tensor Signed-off-by: samsja * fix: fix docstring ndarray and torch tensor Signed-off-by: samsja * fix: fix fix audio url and audio ndarray Signed-off-by: samsja * fix: fix fix audio url and audio ndarray Signed-off-by: samsja * fix: fix video tensor Signed-off-by: samsja * fix: fix video tensor Signed-off-by: samsja * fix: fix audio bytes Signed-off-by: samsja * fix: video and image bytes Signed-off-by: samsja * docs: move typing section Signed-off-by: samsja --------- Signed-off-by: samsja --- docarray/typing/__init__.py | 5 +- docarray/typing/bytes/__init__.py | 4 +- docarray/typing/bytes/audio_bytes.py | 35 ++++---- docarray/typing/bytes/image_bytes.py | 39 ++++----- docarray/typing/bytes/video_bytes.py | 34 ++++---- docarray/typing/tensor/__init__.py | 5 ++ docarray/typing/tensor/audio/audio_ndarray.py | 55 ++++++------ .../tensor/audio/audio_tensorflow_tensor.py | 53 ++++++------ .../typing/tensor/audio/audio_torch_tensor.py | 50 +++++------ docarray/typing/tensor/image/image_ndarray.py | 39 ++++----- .../tensor/image/image_tensorflow_tensor.py | 37 ++++---- .../typing/tensor/image/image_torch_tensor.py | 36 ++++---- docarray/typing/tensor/ndarray.py | 79 ++++++++++-------- docarray/typing/tensor/torch_tensor.py | 61 ++++++++------ docarray/typing/tensor/video/video_ndarray.py | 48 +++++------ .../typing/tensor/video/video_tensor_mixin.py | 55 ++++++------ docarray/typing/url/audio_url.py | 27 +++--- docarray/typing/url/data/05978.jpg | Bin 22715 -> 0 bytes docarray/typing/url/image_url.py | 39 ++++----- docarray/typing/url/text_url.py | 29 +++---- docarray/typing/url/url_3d/mesh_url.py | 37 ++++---- docarray/typing/url/url_3d/point_cloud_url.py | 25 +++--- docarray/typing/url/video_url.py | 70 ++++++++-------- docs/api_references/typing/bytes.md | 6 ++ docs/api_references/typing/id.md | 6 ++ docs/api_references/typing/tensor.md | 3 + docs/api_references/typing/typing.md | 4 - docs/api_references/typing/url.md | 6 ++ file_1.mp4 | Bin 0 -> 8232 bytes file_2.mp4 | Bin 0 -> 8232 bytes 30 files changed, 477 insertions(+), 410 deletions(-) delete mode 100644 docarray/typing/url/data/05978.jpg create mode 100644 docs/api_references/typing/bytes.md create mode 100644 docs/api_references/typing/id.md create mode 100644 docs/api_references/typing/tensor.md delete mode 100644 docs/api_references/typing/typing.md create mode 100644 docs/api_references/typing/url.md create mode 100644 file_1.mp4 create mode 100644 file_2.mp4 diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index f91534c426e..5cd9a6367ad 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -1,6 +1,7 @@ from typing_extensions import TYPE_CHECKING from docarray.typing.bytes import ImageBytes +from docarray.typing.bytes import AudioBytes, ImageBytes, VideoBytes from docarray.typing.id import ID from docarray.typing.tensor import ImageNdArray, ImageTensor from docarray.typing.tensor.audio import AudioNdArray @@ -36,9 +37,11 @@ 'ImageBytes', 'ImageTensor', 'ImageNdArray', + 'ImageBytes', + 'VideoBytes', + 'AudioBytes', ] -from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from docarray.typing.tensor import TensorFlowTensor # noqa: F401 diff --git a/docarray/typing/bytes/__init__.py b/docarray/typing/bytes/__init__.py index 9b622669919..2cf8524bcc0 100644 --- a/docarray/typing/bytes/__init__.py +++ b/docarray/typing/bytes/__init__.py @@ -1,3 +1,5 @@ +from docarray.typing.bytes.audio_bytes import AudioBytes from docarray.typing.bytes.image_bytes import ImageBytes +from docarray.typing.bytes.video_bytes import VideoBytes -__all__ = ['ImageBytes'] +__all__ = ['ImageBytes', 'VideoBytes', 'AudioBytes'] diff --git a/docarray/typing/bytes/audio_bytes.py b/docarray/typing/bytes/audio_bytes.py index 643c753067c..4a0283c8761 100644 --- a/docarray/typing/bytes/audio_bytes.py +++ b/docarray/typing/bytes/audio_bytes.py @@ -47,32 +47,35 @@ def load(self) -> Tuple[np.ndarray, int]: """ Load the Audio from the bytes into a numpy.ndarray Audio tensor - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional + from docarray import BaseDoc + from docarray.typing import AudioUrl, NdArray, AudioBytes - from docarray import BaseDoc - import numpy as np + import numpy as np - from docarray.typing import AudioUrl + class MyAudio(BaseDoc): + url: AudioUrl + tensor: Optional[NdArray] + bytes: Optional[AudioBytes] + frame_rate: Optional[float] - class MyAudio(Document): - url: AudioUrl - tensor: Optional[NdArray] - bytes: Optional[bytes] + doc = MyAudio(url='https://www.kozco.com/tech/piano2.wav') + doc.bytes = doc.url.load_bytes() + doc.tensor, doc.frame_rate = doc.bytes.load() - doc = MyAudio(url="toydata/hello.wav") - doc.bytes = doc.url.load_bytes() - doc.tensor, doc.frame_rate = doc.bytes.load() + # Note this is equivalent to do - # Note this is equivalent to do + doc.tensor, doc.frame_rate = doc.url.load() - doc.tensor, doc.frame_rate = doc.url.load() - - assert isinstance(doc.audio_tensor, np.ndarray) + assert isinstance(doc.tensor, np.ndarray) + ``` + --- :return: np.ndarray representing the Audio as RGB values """ if TYPE_CHECKING: diff --git a/docarray/typing/bytes/image_bytes.py b/docarray/typing/bytes/image_bytes.py index c7d2bf8401d..f8aaf3c6caf 100644 --- a/docarray/typing/bytes/image_bytes.py +++ b/docarray/typing/bytes/image_bytes.py @@ -52,34 +52,35 @@ def load( """ Load the image from the bytes into a numpy.ndarray image tensor - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from docarray import BaseDoc + from docarray.typing import ImageUrl + import numpy as np - from docarray import BaseDoc - from docarray.typing import ImageUrl - import numpy as np + class MyDoc(BaseDoc): + img_url: ImageUrl - class MyDoc(BaseDoc): - img_url: ImageUrl + doc = MyDoc( + img_url="https://upload.wikimedia.org/wikipedia/commons/8/80/" + "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg" + ) - doc = MyDoc( - img_url="https://upload.wikimedia.org/wikipedia/commons/8/80/" - "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg" - ) + img_tensor = doc.img_url.load() + assert isinstance(img_tensor, np.ndarray) - img_tensor = doc.img_url.load() - assert isinstance(img_tensor, np.ndarray) + img_tensor = doc.img_url.load(height=224, width=224) + assert img_tensor.shape == (224, 224, 3) - img_tensor = doc.img_url.load(height=224, width=224) - assert img_tensor.shape == (224, 224, 3) - - layout = ('C', 'W', 'H') - img_tensor = doc.img_url.load(height=100, width=200, axis_layout=layout) - assert img_tensor.shape == (3, 200, 100) + layout = ('C', 'W', 'H') + img_tensor = doc.img_url.load(height=100, width=200, axis_layout=layout) + assert img_tensor.shape == (3, 200, 100) + ``` + --- :param width: width of the image tensor. :param height: height of the image tensor. diff --git a/docarray/typing/bytes/video_bytes.py b/docarray/typing/bytes/video_bytes.py index 31b6cdc97a4..bac2733b8db 100644 --- a/docarray/typing/bytes/video_bytes.py +++ b/docarray/typing/bytes/video_bytes.py @@ -5,9 +5,9 @@ from pydantic import parse_obj_as from pydantic.validators import bytes_validator -from docarray.typing import AudioNdArray, NdArray, VideoNdArray from docarray.typing.abstract_type import AbstractType from docarray.typing.proto_register import _register_proto +from docarray.typing.tensor import AudioNdArray, NdArray, VideoNdArray from docarray.utils._internal.misc import import_library if TYPE_CHECKING: @@ -56,29 +56,33 @@ def load(self, **kwargs) -> VideoLoadResult: (`VideoLoadResult.audio`) and an NdArray containing the key frame indices (`VideoLoadResult.key_frame_indices`). - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from docarray import BaseDoc + from docarray.typing import VideoUrl + import numpy as np - from docarray import BaseDoc - from docarray.typing import VideoUrl - import numpy as np + class MyDoc(BaseDoc): + video_url: VideoUrl - class MyDoc(BaseDoc): - video_url: VideoUrl + doc = MyDoc( + video_url='https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true' + ) - doc = MyDoc(video_url="toydata/mp_.mp4") + video, audio, key_frame_indices = doc.video_url.load() + assert isinstance(video, np.ndarray) + assert isinstance(audio, np.ndarray) + assert isinstance(key_frame_indices, np.ndarray) + ``` + + --- - video, audio, key_frame_indices = doc.video_url.load() - assert isinstance(video, np.ndarray) - assert isinstance(audio, np.ndarray) - assert isinstance(key_frame_indices, np.ndarray) :param kwargs: supports all keyword arguments that are being supported by - av.open() as described in: - https://pyav.org/docs/stable/api/_globals.html?highlight=open#av.open + av.open() as described [here](https://pyav.org/docs/stable/api/_globals.html?highlight=open#av.open) :return: a VideoLoadResult instance with video, audio and keyframe indices """ if TYPE_CHECKING: diff --git a/docarray/typing/tensor/__init__.py b/docarray/typing/tensor/__init__.py index c40feac93b8..7bd6caa9af0 100644 --- a/docarray/typing/tensor/__init__.py +++ b/docarray/typing/tensor/__init__.py @@ -1,9 +1,11 @@ from typing_extensions import TYPE_CHECKING +from docarray.typing.tensor.audio import AudioNdArray from docarray.typing.tensor.embedding import AnyEmbedding, NdArrayEmbedding from docarray.typing.tensor.image import ImageNdArray, ImageTensor from docarray.typing.tensor.ndarray import NdArray from docarray.typing.tensor.tensor import AnyTensor +from docarray.typing.tensor.video import VideoNdArray __all__ = [ 'NdArray', @@ -12,6 +14,8 @@ 'NdArrayEmbedding', 'ImageNdArray', 'ImageTensor', + 'AudioNdArray', + 'VideoNdArray', ] from docarray.utils._internal.misc import import_library @@ -23,6 +27,7 @@ from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 from docarray.typing.tensor.tensorflow_tensor import TensorFlowTensor # noqa: F401 from docarray.typing.tensor.torch_tensor import TorchTensor # noqa: F401 + from docarray.typing.tensor.video import VideoTorchTensor # noqa: F401 torch_tensors = ['TorchEmbedding', 'ImageTorchTensor', 'TorchTensor'] tf_tensors = ['TensorFlowEmbedding', 'TensorFlowTensor', 'ImageTensorFlowTensor'] diff --git a/docarray/typing/tensor/audio/audio_ndarray.py b/docarray/typing/tensor/audio/audio_ndarray.py index bae64ed36c2..f47f40b9d8a 100644 --- a/docarray/typing/tensor/audio/audio_ndarray.py +++ b/docarray/typing/tensor/audio/audio_ndarray.py @@ -10,43 +10,44 @@ class AudioNdArray(AbstractAudioTensor, NdArray): Adds audio-specific features to the tensor. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + from docarray import BaseDoc + from docarray.typing import AudioNdArray, AudioUrl + import numpy as np - from docarray import BaseDoc - from docarray.typing import AudioNdArray, AudioUrl - import numpy as np + class MyAudioDoc(BaseDoc): + title: str + audio_tensor: Optional[AudioNdArray] + url: Optional[AudioUrl] + bytes_: Optional[bytes] - class MyAudioDoc(BaseDoc): - title: str - audio_tensor: Optional[AudioNdArray] - url: Optional[AudioUrl] - bytes_: Optional[bytes] + # from tensor + doc_1 = MyAudioDoc( + title='my_first_audio_doc', + audio_tensor=np.random.rand(1000, 2), + ) - # from tensor - doc_1 = MyAudioDoc( - title='my_first_audio_doc', - audio_tensor=np.random.rand(1000, 2), - ) + doc_1.audio_tensor.save(file_path='/tmp/file_1.wav') + doc_1.bytes_ = doc_1.audio_tensor.to_bytes() - doc_1.audio_tensor.save(file_path='path/to/file_1.wav') - doc_1.bytes_ = doc_1.audio_tensor.to_bytes() + # from url + doc_2 = MyAudioDoc( + title='my_second_audio_doc', + url='https://www.kozco.com/tech/piano2.wav', + ) - # from url - doc_2 = MyAudioDoc( - title='my_second_audio_doc', - url='https://www.kozco.com/tech/piano2.wav', - ) - - doc_2.audio_tensor = doc_2.url.load() - doc_2.audio_tensor.save(file_path='path/to/file_2.wav') - doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + doc_2.audio_tensor, _ = doc_2.url.load() + doc_2.audio_tensor.save(file_path='/tmp/file_2.wav') + doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + ``` + --- """ ... diff --git a/docarray/typing/tensor/audio/audio_tensorflow_tensor.py b/docarray/typing/tensor/audio/audio_tensorflow_tensor.py index a91b5b3183a..497a57f9464 100644 --- a/docarray/typing/tensor/audio/audio_tensorflow_tensor.py +++ b/docarray/typing/tensor/audio/audio_tensorflow_tensor.py @@ -16,44 +16,47 @@ class AudioTensorFlowTensor( Adds audio-specific features to the tensor. - EXAMPLE USAGE - .. code-block:: python + ``` - from typing import Optional + --- + from typing import Optional - import tensorflow as tf - from pydantic import parse_obj_as + import tensorflow as tf + from pydantic import parse_obj_as - from docarray import BaseDoc - from docarray.typing import AudioTensorFlowTensor, AudioUrl + from docarray import BaseDoc + from docarray.typing import AudioTensorFlowTensor, AudioUrl - class MyAudioDoc(BaseDoc): - title: str - audio_tensor: Optional[AudioTensorFlowTensor] - url: Optional[AudioUrl] - bytes_: Optional[bytes] + class MyAudioDoc(BaseDoc): + title: str + audio_tensor: Optional[AudioTensorFlowTensor] + url: Optional[AudioUrl] + bytes_: Optional[bytes] - doc_1 = MyAudioDoc( - title='my_first_audio_doc', - audio_tensor=tf.random.normal((1000, 2)), - ) + doc_1 = MyAudioDoc( + title='my_first_audio_doc', + audio_tensor=tf.random.normal((1000, 2)), + ) - doc_1.audio_tensor.save(file_path='path/to/file_1.wav') - doc_1.bytes_ = doc_1.audio_tensor.to_bytes() + doc_1.audio_tensor.save(file_path='path/to/file_1.wav') + doc_1.bytes_ = doc_1.audio_tensor.to_bytes() - doc_2 = MyAudioDoc( - title='my_second_audio_doc', - url='https://www.kozco.com/tech/piano2.wav', - ) + doc_2 = MyAudioDoc( + title='my_second_audio_doc', + url='https://www.kozco.com/tech/piano2.wav', + ) - doc_2.audio_tensor = doc_2.url.load() - doc_2.audio_tensor.save(file_path='path/to/file_2.wav') - doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + doc_2.audio_tensor = doc_2.url.load() + doc_2.audio_tensor.save(file_path='path/to/file_2.wav') + doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + ``` + + --- """ ... diff --git a/docarray/typing/tensor/audio/audio_torch_tensor.py b/docarray/typing/tensor/audio/audio_torch_tensor.py index 86d09e2212f..532d97a7f8e 100644 --- a/docarray/typing/tensor/audio/audio_torch_tensor.py +++ b/docarray/typing/tensor/audio/audio_torch_tensor.py @@ -10,41 +10,43 @@ class AudioTorchTensor(AbstractAudioTensor, TorchTensor, metaclass=metaTorchAndN Adds audio-specific features to the tensor. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + import torch - import torch + from docarray import BaseDoc + from docarray.typing import AudioTorchTensor, AudioUrl - from docarray import BaseDoc - from docarray.typing import AudioTorchTensor, AudioUrl + class MyAudioDoc(BaseDoc): + title: str + audio_tensor: Optional[AudioTorchTensor] + url: Optional[AudioUrl] + bytes_: Optional[bytes] - class MyAudioDoc(BaseDoc): - title: str - audio_tensor: Optional[AudioTorchTensor] - url: Optional[AudioUrl] - bytes_: Optional[bytes] + doc_1 = MyAudioDoc( + title='my_first_audio_doc', + audio_tensor=torch.zeros(1000, 2), + ) - doc_1 = MyAudioDoc( - title='my_first_audio_doc', - audio_tensor=torch.randn(size=(1000, 2)), - ) + doc_1.audio_tensor.save(file_path='/tmp/file_1.wav') + doc_1.bytes_ = doc_1.audio_tensor.to_bytes() - doc_1.audio_tensor.save(file_path='path/to/file_1.wav') - doc_1.bytes_ = doc_1.audio_tensor.to_bytes() + doc_2 = MyAudioDoc( + title='my_second_audio_doc', + url='https://www.kozco.com/tech/piano2.wav', + ) - doc_2 = MyAudioDoc( - title='my_second_audio_doc', - url='https://www.kozco.com/tech/piano2.wav', - ) + doc_2.audio_tensor, _ = doc_2.url.load() + doc_2.audio_tensor.save(file_path='/tmp/file_2.wav') + doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + ``` - doc_2.audio_tensor = doc_2.url.load() - doc_2.audio_tensor.save(file_path='path/to/file_2.wav') - doc_2.bytes_ = doc_1.audio_tensor.to_bytes() + --- """ ... diff --git a/docarray/typing/tensor/image/image_ndarray.py b/docarray/typing/tensor/image/image_ndarray.py index df304cd1d21..4ac022b300b 100644 --- a/docarray/typing/tensor/image/image_ndarray.py +++ b/docarray/typing/tensor/image/image_ndarray.py @@ -14,34 +14,35 @@ class ImageNdArray(AbstractImageTensor, NdArray): optimized to send over the wire - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + from docarray import BaseDoc + from docarray.typing import ImageNdArray, ImageUrl - from docarray import BaseDoc - from docarray.typing import ImageNdArray, ImageUrl + class MyImageDoc(BaseDoc): + title: str + tensor: Optional[ImageNdArray] + url: Optional[ImageUrl] + bytes: Optional[bytes] - class MyImageDoc(BaseDoc): - title: str - tensor: Optional[ImageNdArray] - url: Optional[ImageUrl] - bytes: Optional[bytes] + # from url + doc = MyImageDoc( + title='my_second_audio_doc', + url="https://upload.wikimedia.org/wikipedia/commons/8/80/" + "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", + ) - # from url - doc = MyImageDoc( - title='my_second_audio_doc', - url="https://upload.wikimedia.org/wikipedia/commons/8/80/" - "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", - ) + doc.tensor = doc.url.load() - doc.tensor = doc.url.load() - - doc.bytes = doc.tensor.to_bytes() + doc.bytes = doc.tensor.to_bytes() + ``` + --- """ ... diff --git a/docarray/typing/tensor/image/image_tensorflow_tensor.py b/docarray/typing/tensor/image/image_tensorflow_tensor.py index 7afcbb38086..95aba9d5337 100644 --- a/docarray/typing/tensor/image/image_tensorflow_tensor.py +++ b/docarray/typing/tensor/image/image_tensorflow_tensor.py @@ -18,31 +18,32 @@ class ImageTensorFlowTensor( optimized to send over the wire - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + from docarray import BaseDoc + from docarray.typing import ImageTensorFlowTensor, ImageUrl - from docarray import BaseDoc - from docarray.typing import ImageTensorFlowTensor, ImageUrl + class MyImageDoc(BaseDoc): + title: str + tensor: Optional[ImageTensorFlowTensor] + url: Optional[ImageUrl] + bytes: Optional[bytes] - class MyImageDoc(BaseDoc): - title: str - tensor: Optional[ImageTensorFlowTensor] - url: Optional[ImageUrl] - bytes: Optional[bytes] + doc = MyImageDoc( + title='my_second_image_doc', + url="https://upload.wikimedia.org/wikipedia/commons/8/80/" + "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", + ) + doc.tensor = doc.url.load() + doc.bytes = doc.tensor.to_bytes() + ``` - doc = MyImageDoc( - title='my_second_image_doc', - url="https://upload.wikimedia.org/wikipedia/commons/8/80/" - "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", - ) - doc.tensor = doc.url.load() - doc.bytes = doc.tensor.to_bytes() - + --- """ ... diff --git a/docarray/typing/tensor/image/image_torch_tensor.py b/docarray/typing/tensor/image/image_torch_tensor.py index 90ecac13750..bc2a8dbc7db 100644 --- a/docarray/typing/tensor/image/image_torch_tensor.py +++ b/docarray/typing/tensor/image/image_torch_tensor.py @@ -16,31 +16,33 @@ class ImageTorchTensor(AbstractImageTensor, TorchTensor, metaclass=metaTorchAndN optimized to send over the wire - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + from docarray import BaseDoc + from docarray.typing import ImageTorchTensor, ImageUrl - from docarray import BaseDoc - from docarray.typing import ImageTorchTensor, ImageUrl + class MyImageDoc(BaseDoc): + title: str + tensor: Optional[ImageTorchTensor] + url: Optional[ImageUrl] + bytes: Optional[bytes] - class MyImageDoc(BaseDoc): - title: str - tensor: Optional[ImageTorchTensor] - url: Optional[ImageUrl] - bytes: Optional[bytes] + doc = MyImageDoc( + title='my_second_image_doc', + url="https://upload.wikimedia.org/wikipedia/commons/8/80/" + "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", + ) - doc = MyImageDoc( - title='my_second_image_doc', - url="https://upload.wikimedia.org/wikipedia/commons/8/80/" - "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg", - ) + doc.tensor = doc.url.load() + doc.bytes = doc.tensor.to_bytes() + ``` - doc.tensor = doc.url.load() - doc.bytes = doc.tensor.to_bytes() + --- """ ... diff --git a/docarray/typing/tensor/ndarray.py b/docarray/typing/tensor/ndarray.py index 0fc41a0c184..49b23ea8e9c 100644 --- a/docarray/typing/tensor/ndarray.py +++ b/docarray/typing/tensor/ndarray.py @@ -46,47 +46,54 @@ class NdArray(np.ndarray, AbstractTensor, Generic[ShapeT]): This type can also be used in a parametrized way, specifying the shape of the array. - EXAMPLE USAGE - - .. code-block:: python - - from docarray import BaseDoc - from docarray.typing import NdArray - import numpy as np - - - class MyDoc(BaseDoc): - arr: NdArray - image_arr: NdArray[3, 224, 224] - square_crop: NdArray[3, 'x', 'x'] - random_image: NdArray[3, ...] # first dimension is fixed, can have arbitrary shape - - - # create a document with tensors - doc = MyDoc( - arr=np.zeros((128,)), - image_arr=np.zeros((3, 224, 224)), - square_crop=np.zeros((3, 64, 64)), - random_image=np.zeros(3, 128, 256), - ) - assert doc.image_arr.shape == (3, 224, 224) - - # automatic shape conversion - doc = MyDoc( - arr=np.zeros((128,)), - image_arr=np.zeros((224, 224, 3)), # will reshape to (3, 224, 224) - square_crop=np.zeros((3, 128, 128)), - random_image=np.zeros(3, 64, 128), - ) - assert doc.image_arr.shape == (3, 224, 224) - - # !! The following will raise an error due to shape mismatch !! + --- + + ```python + from docarray import BaseDoc + from docarray.typing import NdArray + import numpy as np + + + class MyDoc(BaseDoc): + arr: NdArray + image_arr: NdArray[3, 224, 224] + square_crop: NdArray[3, 'x', 'x'] + random_image: NdArray[3, ...] # first dimension is fixed, can have arbitrary shape + + + # create a document with tensors + doc = MyDoc( + arr=np.zeros((128,)), + image_arr=np.zeros((3, 224, 224)), + square_crop=np.zeros((3, 64, 64)), + random_image=np.zeros((3, 128, 256)), + ) + assert doc.image_arr.shape == (3, 224, 224) + + # automatic shape conversion + doc = MyDoc( + arr=np.zeros((128,)), + image_arr=np.zeros((224, 224, 3)), # will reshape to (3, 224, 224) + square_crop=np.zeros((3, 128, 128)), + random_image=np.zeros((3, 64, 128)), + ) + assert doc.image_arr.shape == (3, 224, 224) + + # !! The following will raise an error due to shape mismatch !! + from pydantic import ValidationError + + try: doc = MyDoc( arr=np.zeros((128,)), image_arr=np.zeros((224, 224)), # this will fail validation square_crop=np.zeros((3, 128, 64)), # this will also fail validation - random_image=np.zeros(4, 64, 128), # this will also fail validation + random_image=np.zeros((4, 64, 128)), # this will also fail validation ) + except ValidationError as e: + pass + ``` + + --- """ __parametrized_meta__ = metaNumpy diff --git a/docarray/typing/tensor/torch_tensor.py b/docarray/typing/tensor/torch_tensor.py index e2ece8c84dd..1ec05df0086 100644 --- a/docarray/typing/tensor/torch_tensor.py +++ b/docarray/typing/tensor/torch_tensor.py @@ -53,47 +53,54 @@ class TorchTensor( This type can also be used in a parametrized way, specifying the shape of the tensor. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from docarray import BaseDoc + from docarray.typing import TorchTensor + import torch - from docarray import BaseDoc - from docarray.typing import TorchTensor - import torch + class MyDoc(BaseDoc): + tensor: TorchTensor + image_tensor: TorchTensor[3, 224, 224] + square_crop: TorchTensor[3, 'x', 'x'] + random_image: TorchTensor[ + 3, ... + ] # first dimension is fixed, can have arbitrary shape - class MyDoc(BaseDoc): - tensor: TorchTensor - image_tensor: TorchTensor[3, 224, 224] - square_crop: TorchTensor[3, 'x', 'x'] - random_image: TorchTensor[ - 3, ... - ] # first dimension is fixed, can have arbitrary shape + # create a document with tensors + doc = MyDoc( + tensor=torch.zeros(128), + image_tensor=torch.zeros(3, 224, 224), + square_crop=torch.zeros(3, 64, 64), + random_image=torch.zeros(3, 128, 256), + ) - # create a document with tensors - doc = MyDoc( - tensor=torch.zeros(128), - image_tensor=torch.zeros(3, 224, 224), - square_crop=torch.zeros(3, 64, 64), - random_image=torch.zeros(3, 128, 256), - ) + # automatic shape conversion + doc = MyDoc( + tensor=torch.zeros(128), + image_tensor=torch.zeros(224, 224, 3), # will reshape to (3, 224, 224) + square_crop=torch.zeros(3, 128, 128), + random_image=torch.zeros(3, 64, 128), + ) - # automatic shape conversion - doc = MyDoc( - tensor=torch.zeros(128), - image_tensor=torch.zeros(224, 224, 3), # will reshape to (3, 224, 224) - square_crop=torch.zeros(3, 128, 128), - random_image=torch.zeros(3, 64, 128), - ) + # !! The following will raise an error due to shape mismatch !! + from pydantic import ValidationError - # !! The following will raise an error due to shape mismatch !! + try: doc = MyDoc( tensor=torch.zeros(128), image_tensor=torch.zeros(224, 224), # this will fail validation square_crop=torch.zeros(3, 128, 64), # this will also fail validation random_image=torch.zeros(4, 64, 128), # this will also fail validation ) + except ValidationError as e: + pass + ``` + + --- """ __parametrized_meta__ = metaTorchAndNode diff --git a/docarray/typing/tensor/video/video_ndarray.py b/docarray/typing/tensor/video/video_ndarray.py index d52e9a1ae28..f6b3cd8d3b1 100644 --- a/docarray/typing/tensor/video/video_ndarray.py +++ b/docarray/typing/tensor/video/video_ndarray.py @@ -19,41 +19,41 @@ class VideoNdArray(NdArray, VideoTensorMixin): Subclass of NdArray, to represent a video tensor. Adds video-specific features to the tensor. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional - from typing import Optional + import numpy as np + from pydantic import parse_obj_as - import numpy as np - from pydantic import parse_obj_as + from docarray import BaseDoc + from docarray.typing import VideoNdArray, VideoUrl - from docarray import BaseDoc - from docarray.typing import VideoNdArray, VideoUrl + class MyVideoDoc(BaseDoc): + title: str + url: Optional[VideoUrl] + video_tensor: Optional[VideoNdArray] - class MyVideoDoc(BaseDoc): - title: str - url: Optional[VideoUrl] - video_tensor: Optional[VideoNdArray] + doc_1 = MyVideoDoc( + title='my_first_video_doc', + video_tensor=np.random.random((100, 224, 224, 3)), + ) - doc_1 = MyVideoDoc( - title='my_first_video_doc', - video_tensor=np.random.random((100, 224, 224, 3)), - ) + doc_1.video_tensor.save(file_path='file_1.mp4') - doc_1.video_tensor.save(file_path='file_1.mp4') + doc_2 = MyVideoDoc( + title='my_second_video_doc', + url='file_1.mp4', + ) + doc_2.video_tensor = parse_obj_as(VideoNdArray, doc_2.url.load().video) + doc_2.video_tensor.save(file_path='file_2.mp4') + ``` - doc_2 = MyVideoDoc( - title='my_second_video_doc', - url='https://www.kozco.com/tech/piano2.wav', - ) - - doc_2.video_tensor = parse_obj_as(VideoNdArray, doc_2.url.load().video) - doc_2.video_tensor.save(file_path='file_2.mp4') - + --- """ @classmethod diff --git a/docarray/typing/tensor/video/video_tensor_mixin.py b/docarray/typing/tensor/video/video_tensor_mixin.py index 7cdc1b82ff6..4ce6a1b7260 100644 --- a/docarray/typing/tensor/video/video_tensor_mixin.py +++ b/docarray/typing/tensor/video/video_tensor_mixin.py @@ -38,6 +38,34 @@ def save( """ Save video tensor to a .mp4 file. + --- + + ```python + import numpy as np + + from docarray import BaseDoc + from docarray.typing.tensor.audio.audio_tensor import AudioTensor + from docarray.typing.tensor.video.video_tensor import VideoTensor + + + class MyDoc(BaseDoc): + video_tensor: VideoTensor + audio_tensor: AudioTensor + + + doc = MyDoc( + video_tensor=np.random.randint(low=0, high=256, size=(10, 200, 300, 3)), + audio_tensor=np.random.randn(100, 1, 1024).astype("float32"), + ) + + doc.video_tensor.save( + file_path="/tmp/mp_.mp4", + audio_tensor=doc.audio_tensor, + audio_format="flt", + ) + ``` + + --- :param file_path: path to a .mp4 file. If file is a string, open the file by that name, otherwise treat it as a file-like object. :param audio_tensor: AudioTensor containing the video's soundtrack. @@ -47,33 +75,6 @@ def save( :param audio_codec: the name of an audio decoder/encoder. :param audio_format: the name of one of the audio formats supported by PyAV, such as 'flt', 'fltp', 's16' or 's16p'. - - EXAMPLE USAGE - - .. code-block:: python - import numpy as np - - from docarray import BaseDoc - from docarray.typing.tensor.audio.audio_tensor import AudioTensor - from docarray.typing.tensor.video.video_tensor import VideoTensor - - - class MyDoc(BaseDoc): - video_tensor: VideoTensor - audio_tensor: AudioTensor - - - doc = MyDoc( - video_tensor=np.random.randint(low=0, high=256, size=(10, 200, 300, 3)), - audio_tensor=np.random.randn(100, 1, 1024).astype("float32"), - ) - - doc.video_tensor.save( - file_path="toydata/mp_.mp4", - audio_tensor=doc.audio_tensor, - audio_format="flt", - ) - """ if TYPE_CHECKING: import av diff --git a/docarray/typing/url/audio_url.py b/docarray/typing/url/audio_url.py index dbfdd84029b..bab24b518cf 100644 --- a/docarray/typing/url/audio_url.py +++ b/docarray/typing/url/audio_url.py @@ -48,27 +48,30 @@ def load(self: T) -> Tuple[np.ndarray, int]: """ Load the data from the url into an AudioNdArray. - :return: AudioNdArray representing the audio file content. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from typing import Optional + from docarray import BaseDoc + import numpy as np - from docarray import BaseDoc - import numpy as np + from docarray.typing import AudioUrl, AudioNdArray - from docarray.typing import AudioUrl + class MyDoc(BaseDoc): + audio_url: AudioUrl + audio_tensor: Optional[AudioNdArray] - class MyDoc(Document): - audio_url: AudioUrl - audio_tensor: AudioNdArray + doc = MyDoc(audio_url='https://www.kozco.com/tech/piano2.wav') + doc.audio_tensor, _ = doc.audio_url.load() + assert isinstance(doc.audio_tensor, np.ndarray) + ``` - doc = MyDoc(audio_url="toydata/hello.wav") - doc.audio_tensor, doc.frame_rate = doc.audio_url.load() - assert isinstance(doc.audio_tensor, np.ndarray) + --- + :return: AudioNdArray representing the audio file content. """ bytes_ = AudioBytes(self.load_bytes()) return bytes_.load() diff --git a/docarray/typing/url/data/05978.jpg b/docarray/typing/url/data/05978.jpg deleted file mode 100644 index 3f0bf32e01d4f2737c6b5898fe0203499c6a04f9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 22715 zcmbTdd03M9|2|67R8v;AnWZVyVoteDxzJ+rohdVO9H&LxQl`w6kSTLVp3f|0X2zJQ zDU(c@nhTN(kfJ=LsVS1FsR;t2VWlFA$rAAJd;0v&b-=%fp^N8|fQR>bzwXz4 zzuu;|rU}HllgE9JBbF>#g189(Lzo1JqlmBImrIs>xqkKK@%7R#*JWRSz4Ytl%a$+y z?{meTEArJ`X#0p2qXfrWZ8e4w*>KjJeGX5^y_8ISHR6MgFnDn2RDD| zQn=B};1@! z*>cNPE4$tH-|gA!=I((yaPZL4W8TMoPWYZYb2bnY6dV#7c`+(FCN?fU@oG|XN@`mA zpMPDuo^>Pp=AHbz1%*ZTic21rJu0uLd|XxC(AdOavYs}#w0FGd?CO5m(>plyW_V=u z?HFG&IrUCD{rd#P0L5 z=Pz^pcJ1FL#PCVB2{Dv;EpxF;gg zLS1S?{Lu~#fLkV?4qO%fh+h2>W!5~mdH2&hQ&5WoNk&qc5NqYem7FxDkzqo_W;z5O zB478~8_)OTy4=hEkncos0Mdxpuq9OY)6ExvA4}J2txbrpZz(f+5+%s7el;*`g)W?Y zHS{dq!Qa}2ry-KrD&x$1(&1EV=o`8Ikl_GF%TrdW&ui_=Kd8TV%f>}17JP6uT>{nf z)n3OxUL?$$5auF0=uX-)4m>{!#$qkQ1w19T^KWgtAJz$<E-&yW#VNe7u`o@1z9AfRKq>KoSW&|~VX|7& zlQ)ZIQVpBHiIh(Ax`-m%;*8n#;==@Nt zyK^ph$I+BN6C$SZwz~=O9~o9%QaIV~pLwQ-DYR>iW2Goh2!JeTS>R>->YIP-D1fwP zCosi$bYWW;P2F616t}AKFURd&vOBWzL7l&ApTvpRxD#Z-Bx}Rr(Ve=Kkkm5C zG$j)W9a%0nq(9Bb_-6hzj`FeOz*IIxs$&7GXs3dIYDf#vgYrfbV$FAtf7e*HleG?& z9l)yY3#A9(S&k?14!zHJ$YjaD<)0W~*9f}lfn&54T%)uU`~iqE*X{uf)}zY7nf{~N z4*fF24lSYjryOe!y5diteE&ETqJ@|;W_i-+P(i1Q|dc5lE7 z6JiY2xo%wRcP8JzvBXGl!DBGEkn%viDAX|Enkn`=(rDo6`9_@aE-*0>Fh-vGxu{Zm zbTWnPEF=F^3+7t8M;V-ccxS!W?m_GQ6lCu}6JjvFuf>>+o}h#k zJEA8_fF+|)i@|O{U_#()nie>Or(EVgLz(aC|8%Anb-txve-sIAxqtUc>o!u)bPDxJ zkzwQf`{aJDedV*%6aa2F*@SpGEWt8HuE9U_r!n`}Ap*%A>NFu%;>E#2BL!DsupOL^ z+e*?1ne;Nqok-I9={-}u2)2euC9m|)@(z_wbJ{o!&Cr!W@5F}9N7aMIQZ(+ZVWSDL z%AvC{o>$h!#m+yugLFv|>W4r4);L*e$J=APz`#bz6a7j~k=rq@Gr^u{_Lc=0Mijpl zJqs8^KLZp$_b)YsYgg*swHu$Ol)QhD>3-(4;{BhOn)H=T`F?LMs&?J^W|%o2EFfqK z0Je9439;fUZN6_aJUz#W76TP+SY~)C_e9H0^U2t$fklgG6N2;T_Dy5i@9`IqYoEV* zB$zI3NC7Mp-@86OI*TjPMe1x*;6eD>gvdXpbTQs#gGSWbATd9}gec@Jg>SpWgy{J< zH6QBr7ejx?4QtdCh<2KDoBrZAYb)Gcfhv>F4?o%4KH{g>ce1W;TCgje#5x}s)eAs2 zvlXj;rfr)9{$oNk+TY^r(;niF;Lmh?b`1~lCQ%*;o4>&Bvl4J8Iu_JA0y2UUl!(d| zlt9kF)f}66yR1`1$DXL7afHd=FXF|HRA;%Ggq_3>Z~1)@YuI!KG7E??n?bIEF)56j zk&`*Y3@sylwwBS@GS?pm3?pJk(mAYZIIwjw@1m;A(5M7*h^Hn4HiV~kbe7t;ykX>Y z3oln;g=s@4#GMN$=vs&-sEC#C(N=rd|MG8Qd&+_vRTg+~dovjOx`R>Rl?Jqh9{2ng z(_9i)5oZ^jSIlxfoB)q-dhxasGXn9n36WfzNl6=9{?WsP*Z@O_VR@6qoKsUYvltbT z2F?_^EaIRcV2<%*Yd!sHfL?}H$KQDsb6t`@jdhhK>8r(aL*CL}3G+BXM*w6fEc?6v>19 z2@}SX{E@@OwYxBJm}8>cXrULtiV#|9wEjqKE$G@%4X=YRXj#a>BUj&si?PF`(D6ZnlVoI<{$%C@Iyl1Jn-Ht< zWy8qWBzECJ!$FFFtO)9V4i%Pr2o9&E_-*blW*P**1a>ccqv~gsy9-7~ElQ8u4Xiwz|jK#e`^JypFfJ(@_=gA4M9e7DN_o2nSmUtu;_D zq>Qg9n;33>S}^A_T!P+Lj;kU?37R?Zd?%OjeI$P)j~n1={)0}uqS#gf+;k#!<0jeG z{rj;S$9$OnCtX|!d6{NJ*7q>=GG9b~;->Jhm1u>ZSnHfX*8ilf>xKicd`(BW_U6ar zF<}V!`N$~OIHCtTvDeeRu04!r&XKE)g$qzf8oO|_3Gq8%ZbC$bm0Du0h)0wd%A|Nr zYNW`SVJJOa@}tvpZn*dECUL$(b(SfE$6rvC$W|-zz;~p zDZ+KC;uJEMy8ddogW6Te>&lUc=dOn*U|BS3k(X4!mM);%G`M%bH4te+Bp}}|Rlj{r zT4}hVWn7+g$c%>uZBF;jhCr{c{o71%xzb4N?HzuPUW=Z12q=CD(C(q%B)aMk8|;;r zNLz+!b6)+`UiUN9Xx4t?$7zb(SyrQ>B?4Q)7;Fx*K`whRLH*irNqIG0-Ii4Qf&W*_ ztsoUo#TW-1+!(56XCKeaYeN#w&qUU3v!weZjouB5dVes)BKM*OQeL1}$ydF3S+ zlHq~p5nw#|PZNUK)i#0r3v47YMZ4eZY>i&HL<=bkvDabT33B2acprp!Ozs!gEclph z;qtHO>!+o9@3ZPNGw2!e2^@LNE#rgsj@nwaG8Mef@R|y7tfav4kAG9|WllfNJi2m#Bztc{ls6V3 zSLJMVP?MoMw+PyAJq75C*bVl^vpGx{$jw&ir~F&r)_}=!$yvxnyN0+72LBCj(H99D zVxXJeVu&*5Ah&T^FPKlr;}A+Y*H3Bscc1p`hgN+{PSme8{F2dA5Q$xR?j?23$Txib z`cnO3_@d^$K^D3 zsPDV`VH%uiAPDa3Z5gz?+RQazCze9V$*>?emB;8F5heo(3#vC)`sl>W(w5K2 z;^lYqx~-UF1d9~6T!;&NKx>9ifgI-dX~`#;N)uw+N%o_Bwo@vIXL+;ff(j?2gTiGb#*;Z z8whFoKuv$&@MozaMKW6S?^}@_;~yZ(d(0xg#&LgB29ZHvs|=uGuda30#?v$HLZ-dB zw5eW879l;xNgY-@ABnbS9DGE189evU8x#93N;M1!!?`8|k?|V@W9T0w-9QI${?~!p z5jIiW`g)$A=7fv)kA6suEx|X;6{}ye!+MX8gQD=^HN$^7*OxPz(0c){O93Fls%zfC zTcds*^+o2pr-T&;4EFEjT`prgJ3~Vof@AfyG7Q!oFO4h9n%4arf|l0KiD!4gLsjU` zXS^dFHc}1eC2#cp5M>uBfS2UiP(6>Y76fU57SNbW8jT=2D1NTo`|(U1Mn&XYsOS`0 znGj}pI#T;lzr%!Re>ntmjq|HUoHq0jHp&FnnL-g(2xE2}3$No~i);ALm7AiRLy(7y zfBWZwZG6nsD{2J~zm2>vw<0&>f{&hE(ztjr14nNR_WCd?#eR;%#-s|=2xX;V*PV#n z+02oH$(@_9d+Ex|5dk}eXYe!5ESL}*N!`>9@I|+KWmqsTD9Z7F9?DxmlU%Ydo#3LqLoMDt3&5R|B z-2O3_sh_iBc9)y~IgJVCXbPH(j8YR~vZgO7U^$MoffJilv+_YPFnmn8zXoVyYe|bK zKI>05_n$Gx6m@QNN}8$=LsE+}JXggMV%aPc!Y1_1falNZ5mg)=RYbVT)sJEe!i?qB z=~;g=DEi%_wM}CsJtXXr$#FKAGabs)`_~8zjlS<)6kJOoOh~3_&F2s3D-X)RdK1Fn z7-mB713F0GXb(HxuxY|?~>=4mU!e(auGDw)LI-_Ra=VrYhsT12+DNA}V_7Q9CX{YXSdt*Wz9d?we61Ky! z-McP8o%Uq|K%Llz%}+$G3RITNgkXR$HeJ$Yoz|ayO*gtm>lwPGlVkx^-K8yhd`W38 zTRIsXmK}6;GYk3&(Da4j)H~J7jQOn_Y0yir6-p|cax}T;hJ`lXQK_d+x5jz$cyL3d zBz`e_x&CJnpKi`Tvqwju+RwTx}AS_L-QJX93vL*c+`QmUN z<%O=}w@1PSmU`Dftg+$vMC2dOGDUlmC4n5a+ZBul%-O5#F1>{O^I@NNG`86Iw zvBJvXQ)|BcgO0RI-(QV6*ojMFH&4Ckn~A=nUOXS&{|eOF2N^bzpNZ?cpREa#jIb0M z%m~9lzIlA+ROZde1Crnp{-8@SRm(bRaG!5M8!Ri^!Pv7S=$P=+2Y_5!(g(gbyhg5! zPNT9=#%up^L}wV>ANH$p9>vlqI8$2^FPH95BX44adb3=+uU~;cJ1Md2y>)VL`vXoOBL&7MEaOI2~OQGtN5|Xtnu$YfH(2hbg*W z{p*27LtsWg%bhu=HtCe(_s5hvMkAaXBdC)AYy=^blA%UndtH1A&@~goy z(u@HV=~93-q)Tw%_quAc++|&i{}ASQZdo|B^$xBIf{9J_IM0Omm#%atl@gEWJ|$;j zqzNuoE}~kj+hv0wxf_!63}H2r#+j@S=NtmYeD2j(eE4YQ&h6fH$49gNOtM_=0@JDV z3KQb5bjy8$9chUtpo@(x_@+y+hU6Br@a5BEg)uPU#qRx%vxtlP?CI)??aR}pn6WN_ z`n{vwZ=f^ssu5Un(H?V7NI{PkQ=-Kvk`RPRNOsLBEhgrv>hxwpzNSjk+7C-xbHg94 zfUF0tjkv=OQFjig8}T}(433ual6rz94DT`Zy}+IAyID+4fuRNmeU5)$wddG}+}NMJ zlv1=J`$!Ydy)=6p&vry8b=W@-&0}a6^TO{D2ix`#eQaI#bWCoSA2al5>-8IFTF}?$ zH?&)4@4UX7&>(1Nbi-=;_2Le!t@cmJUG?Y2SoNW!jzbB@SWgRra0!q`Q=_K5DBYM9 z9G3eLj&f$k2OiyaFwH&>y~3npnnVYm3}Ft3Ah~UC!6fh+b3Rn>Sad+m$oz-==j7_^ zTZm|88S*cKq2=~WyOCCrTXtu&N{DV5o(Go`oGJz*8+}UxoG>R?#$h28+A5f2=BUvd z{V%WSOIg=AC*D*oPGTD6ZQ3Lf}L+(7smn_ThGA+uU_ zK)pX9dR*Lzebe0muDS2bR%PAni4z~No#F8ul2-v@9${9&{sFfpI$7y|DL z_ty0&4d*}{oT1*$jKBF7M#1(=q)wR6b}9}+nr=Lt-u=`vY_T@wP@GTUX7qjPsRPX~x%gi519VP*g{O%oxh7K&53@ zz-q&0@=v&MRLAE_E!TCx$hmXE9!7DIHt3QA%Q#)@^|nm@lh*3yceFKGKB9`S=An1T z1qQTxP!;+wVy{^5gosZ~l0bt&`ZHnOR`Y46IPvpMC`w@c9sYqw*Lih2$?ua@?{uKF zLc9oO%p2vJnmdY|kG~23eC*@)yvTU&rmeKh{K=PD15M#Kt?eB!R^Tt7gS?3sldu8o)qquPtyB=KhXS8I~wg({iJm&p5W0Z z{`KbaYXxBmGEg;|EG~jYr)Hq=z8V8MMN85sXhbi3 zKzb#*IMGZ^MlytDc_ z1C;^k-;earuW9({ z7TYRZz=`G$ze8JiS%#(N`%-*H+(wL_exhyW^@?RcTMOsm8@CfzEkJn8XQew-`zkymIJg(?-O<{_(3-zV9hto?ag)PcbEUVDkX%zI)b$$}U)+PY+i z+vN~pTA>)lyL_paW((^{zvK0c>@qFGa0I|d+BD&9IvzH$hr?2!YlgtZ6oUm$a-3qU zdT2P+$c!qE!h5Mw9}3w7`A#@s#cT7i%^7(7#VoByOCr>-sab5b38+0?xl>6DMAnQJ zxXJ;o{p~*>Ph*W2@`Hyc(!Kk}T-j9!xk!bEfzu^6)O@RsMq}vS(FGVdzFyxLfP}`y z=4ZGen@3EDS3n+%Sk1!@RTS(x9Goc}p5aQe-w#2KVI{*x&uTj2ny zW$43VSePyNiK>tRSqk!`R$2)&=I37Z-qBZ)_mGPJS^0NrlyT`gPM~!H3W1l1&H~5w_vDSm9Fqh>n3N=~v@aZK&;`CbWm_tKG1KJ!?XA z2Cam@+D(RBF`|nBLf!;>H&$;#Tmeyrl{hr>7-Hl&5=>J{YZR<1)Dn>O-nXi)WPg7`M>J-MXs&ZXESt)WLHKLqnC) z_POLu+FYPl-~w_wm4nrP1Yz+j(Pt>SB+9q$VidO63ox#LzNTq#*@MKNDVVg~xdqhs z*-^S;4wen^l5Nf3)S{E?EMb^9YeFmx8O4TVP68~OJk&_!Hg@3k^>FquP&>b~C65z; znifEE!}1oq2VcdHl_i{!tpeS!u}|+D$7;<`+@BRPMrnEFT!wpGK?-RR4X@&A#_Ow< zjSDm?SMN0}-b(}(cCh3F7tR@n1f14+quDUQ?uH(TZ15f*@5xKoMu?Z8-; zHv{Cug1UaV|3+mHRdnb(uK9s0F9u64Hs0~yyXnr>UBEt)swP2KeY>hWZJ(|`O;v3W zctr_<(}%<#E5w0A2_@dx!eFW#cm}XT`3dD|z=g-xg5nK-qx-nf0w8w(2W*Yw9=P&y zD#?GtnA5PWlv#^KD_vayhQYmEt2x;Wm@^C98(!8l1G6RbkVem#vAWjSv7vPj<3UkM z4}lFMMfy|fjZpJ{SXOnqvN$lcXS!|}J1GeKIRf_Y0#7VZsc07APe)H*MaqkH;^4hT z^2^rjeEIQ#{smwP)Vq@cQ{_MB`zUvn{M6q}8;yhQw%ag8W$0b>=d}Cr0nbR^4M?s< zhsid4-cpouPlWTq-WH)syTg^iXiD^Q#R?|#~ujs@3tx8ADHmV}N zt(>bjbbq89tXPsT9zop)y=^|O`jlxkEeYqy6?CX?*k~Jf=nh^do{l#hgjvfu$hMr=!f7yHQc`ds0)m9%k=9Z1OJFoLf!1ggPB9p%u2fuB#bSJ)K76qFiMRo z)+h)i=-XDr3_vunNQUfUYhgluSOxj@Yj*>`cfgi%_Yo9Kggy^j6pyY!3x3n4i92;f@Oa&IN<<` zYg0||&3CNGt@Yfe$N2v%>b2A-c8q#FiQ;^!9(V~8F=9Z28% z2~Sdg9KzhKF%05|rn347>Mydun#7T+xw5^~-m4S8HFiZ#9Y@FN-b)B`!-~u!;ADFrz<4=rX=s)=h9 znI^=IR@8*Xce}vyo`xA@9Zi<%xhR8~w&r!~<%yaamzCqBS4it7>+mpNHimLy*y}}& zdplFUcm?1(JcW67NFX@jwQL*zR1I%WB5+kd7i2CW(dW|A;(a8r0I`dNDR}P!$;{hU z+`;ap8QkT_ruTE+lj8rhB++x5a<2$N6{0E!pD4g`{v8I}w&`ycuQ{1zT!=9L?eWEn z$D#zrna_LOtAeuJE)v&VI&$tv$eAjoWvh$C7}VLa$jWL8OTII$I<@UYn#VeQ_ z{>)bBk2Mbw4}sClVEzHin7^qhM!v4@+b)2c|KTaYydM*|D}DFv&vlInER@^|MybLR zloIqjoU3s#tfrdoO!sh?DV9J}{G+mVj@m5-C=qxE9f1fg+87kc?x2sJA7(Z{3r_@ih% zI5s4?#jBV_T6_xQo!knL3RC=cbQ@)T>;gcjhpn(Cs&+4R4*##9ClOHkXnJ$4oUZ33 zqU1nU;{Li}J?J5pKzkLy4QxI28U32aWwF68>lcg;<8Y%qRt@tMx6?j{69RQaW;D!z zIsRJ7Wd9i9M;1r1ct7hV!=*8zw51#7lJ)V;W;u|ul zdQh<$ULA$t4_-?l5y@WJfBpP`mn*iz<_$`=ru(fO)~O&F)?bYuEdc#yy8YIoj>2;d z(?>3?JulA~ka{&KH@>xT3U^NKVi(C_^9}Tn>t`Abu5=*It#I9ZE=CyQY=xU0a3Z*CYFb+5MfpaYA z2=?%q6<(1uRg-%%oa1b!;+Yw(RQp61p32n$1#27sj+#$Z-BE4LztC5kJKf}~GRg~E z2~$O|@-;zi7p4)9Bu)x|gV==rJ(vJkhppDVpF|lp`%TOjsA~?xEx@gD8%X=j6YGG| z`$7G@@V7*|g-b)Kwr|SpB(b%-o|rszo`P$^tHgb&ETdQsP}MjUS?y;mAg+O{e_9*0 z+o$jWr&+A5pwf}ausuOi)fgvG2+fSPrm~?`LW$h^XqxMXh0Gc#WEVIy2IRnWzVjS; z&T@*kF&HL+`Kdtn=RDBr}pI?inE(J-Is2G05@p zOCu~G{b6H3KGAwhus282<2*zxb=l2St zweosnS^FrGPhFQFAo{VdbYuVL2MM@Nhlg$Lv;t6YQjh&jyVY;KR|xVRRgr?qbzfo3 zhBl`jTFD#}5syQ!3ti;CHrM9j#hSjsQ|GV-n~vHJydqzM{8QRYmi;y}DD?e`6Y3tj z8(R$U1>Tp>YRQB(hMh|41f6U+rJ#4VU|$fH#RHMUP2t`NweQE+)8U|2E)rq`>9SLB z_=wcQ*Y;OT#ntqm8JUWKCS&wy@`w3dS3+CB0-3@QgDv`k2&{ z-pMS`pGw_pt)daC;;H9bj_F7tV2TEu@H@Aqg*0mRT`9*eY)02+MHP9;wFkefv`X1} zw~=KO@5aG7+N4n~#4v9k=Zh6~^a?ayzEvC4jb4@H;DiiRP_N@Z>CNKGhuzbrLUzZ# z3-Q84MGL1L0Y|_UFEC(HTr@(`tU$GINdoIx0%rz}5|%v)CwP6Lh>}M%4tY2B7=)WfQXJssDoIo0gRTm`@HCW`kBKAoiPPQTF@R(of%m+iuE$9*`;`20syntTa8=ujD zcI|(xx`1xYgva{pac%gK(HpyzPR6EN%=*lS!aTTYNf$;in8AWQeV~5{TNdV)oLB7G zFEUP(4fAvgZ~?groyA!bDrW}IJ6B?;@AzESJRLd`VVuY~nBS3{EOKhTZQ~oYz%+mv zDO?ROCd9h@bKwYQ?`f}&EXCyqf08xiRBd>9$V2|a|7ma~3U>H|>1|*9A52WZMZd#= zPj9_rQ6K7~kYR`pfFnK-D2(bgiOAjW-?9w%pkz0Ru2U7ONO!v>3 z>34`oU1)|Gkzo9{iMly{KuZd{XE>NtE-<#t{*ph(TV;Guf)zUc#uU~}JIhY!$sn&zWl2V}9}FbaZN{O6KiIr+Vt@dHGOdI0(^3d;uGEJn#1Fpc%aYmh zybDps3xy#RNC&gg)|BoDb714F^>#&jcZ*YxLj~-H%#mctn6ALi)6%?E6pT&+)%1FEx~cVWNY$sDQ_3D;}A>lrzz5lHN)we4)NQ3Nz!yZQeZe@ zJ}*e&=WDK@^kR<@{TCWZ!;$*aMfGq2sX0$_cCg@qs<`~AkWr=!Nv?*+PKqsQ+vxwM z^<%B*l{8Y|_3`vB{jg+&cIa&RRzK0hKZ5^OPcfW@j6` z?P8q&>1#N;Kx8MI5Edi~G-~_J$S0S#$j*|lLKC2l!HHlGSC0=~0yxk#=%^YVai!i~ z8#!5Y%j-yL%rV90qKbfOkE)HKb&tf&(HLlV-JHk%Qyph%E{%rn{2R}@XHQjboJWtz zyU~i%_!Z9;g4ErDNjDD^k`qh2z`lKO7ok;v8{DomH&`4!ormY}l-r94$|J92@S4ob zoH%T*-OFQU#YAT-N{1tJ2%_dYEVRTLUMh4^f{K`?s2EL50JH_;J*+R-cB^Px^gp7x z$H2sCxNbAxs4?sr;0P+VP|1Me3^|L+pk=k&xJ+8gwn_rlcM$Avep}B1N5d&ssWI%R z+>m1M8eB%niKSPgCnk`qAc2+rT!@4hpeAdxCX8X;`IGot$xM;Wz#~=xEKfEvY-r7b z0zQn;e%_g4^Jw=($L0%rdr^3UrBsJK)7|y!hw~0Cm*2BW-@nhAo6?&TNyZ0WYlOrQ zt@ne_nGy9N*(++&uSI4ygH6dJgY$tvK|vub6>xA%yG)3sBs6i^sGE3Os$Al8!!7t~ z=E;>-D)sZSTx_y@k*csVeglugDuM}d{g`rRrNVH6tq0ZSPv}ALe^(ifD3P7!Z~yEn zA!NxyeFkN}j@Sk40fz9_#Acax=5h5$J&D>xVBro$bHQaec$t zp&jy5Xl{o;dKx#D;C}C+Dq)w7gIVA#p!$ZD2l_m#F4%2FGJjS6()sJ2H zet?SjRKqo2^^s{=w68W4`u zQuNyB0cf0#Ds1>s6q*LM!_|7w;fM$ScBuQEr>)I}HFX+&tymuP=S{Lajaz+M2kQn5sn@b${GG1R zi(Zkg4tN1O-Y70%Xb zRvB&A4W9ZhT``y=C1@@75bdLp=KVpUgOqHB{dIqLmt9f%dRG^ShiL*<366B%2flVv zxlo$baMc1uKm3@VK#7TQ=XO}N*ON5qjR_EPp{28ptVbh7dfUzBn zetXNtT)KN`&Z$;bcD>^#KARJ#Nr=|LKJaASolPl?x7(+AtzDTf^(47|xMzejheEb z23wmcvcZrey>)W2bGK@^2xc;DtTV(Pu{^Z9xm?@dF$&u&;pns3v9805w&FRdxGID|t!tI4a>HXms29g4%i#bH(_PpcHq> z$#<}3E3k)g$%XO!gv79M>OjJ!^Zr36!!UU$Hzr-r#C);4g*2!5zS#|KdlJ>;-NPj= zl5sISore|jb{3Ol^Y-t?qcNAIM+Ge%!Sm{U`wO!OLQph>ENVOHJ=2a+3&) zX5pJi)&oNo*$JS!2Zxr-7Z}Nsj3e$4=DJ^FVnv~w01K#`UeH`O7vM6Q?BjD8t(zI^!2GgyH)v%{aV5c%j9I% zYVqv9AzLj|GzA4XIXJddU8l8IrabrXNbh>mg1u3D0qZf$3GwWNz7yyf3vk7IqQsx| zAMlcyc;iP5_<^)*XtEVgO@faVU=3+0fH+VUQS?|G7o>!r8-BRF`sRS&Qdtx{Z`7^7 zmy^roM4#6eo2Pf~p}J*$;7(@i-Sz(QDWvVjk|g|kui@W#8-)KOZWn|lqKpgEK|R1S zVwg>>c#P#tgOTi^aYWt%>ODr_^@{*-0et%om?sk4kXurd1GZ>WFoRN7OUqD2VC!`c zG7e^xV!6{5a7EJ7^TV5Ge8ykT2kJN)9;}rF^>?grbd1_+t3K@3KBYW6b4KqoT)O0S zQqEy6?ZPW#NJ8!A15Gl{#mle0R5jSrk15u&YJ4)&DrUdu*JRcc$@eMAusT^tl@S>o z29$Egxep3Kh#)Pm+j*9j+f0$_#ah*9VA*r`(#MTQ7mg1$-R3Urm(iW;Kb+5eN&O+! z;}z1`y5|A*`xj5Rdr`ghaG0Njy-RseS!zmHrftBJIRa(=5gYupS0(-{eepi+R`RHv zpf2~m9TpO09M6F;Z2U>p*4CGJVdR2y>x)z{RggSNKJEHm>P1=M9H+Y9MS%-Nc2OiD zmi>ZyJNy8w%Q0bmD`eAF=sgZ&554DNU3+s+pB3(4$|aDCe!3Ogk*>aP{FnJM1TWC^ zNQrFtp|$j_&l6boK|a#eWYRpu3O@GxP! z{xIsZCl&=;IG=ccwi@<4|0(BgJaq5napd}1(2O#O9r?al4uMQasIU z@4UWu_#Lp`gqV0j+N%x1C9Prox#)lLL>TvGhkp6=RuqtaJeElp!AH7o2VfG7<7v0C zvfkielIp=H&GR;lphLW&jH9ktJPd3vwOkD-N9u^Z%Jx!x#@RLg5WM%XqROAS9HHh8pJY#QWpwKa5LTW)+0m;d#aB*i$k(QR4yZfs zBU@~_bj~iK+G9AK(=YbgWhnT6!dtS|^T`*T46EJr9>R&>=ur`V<>vnoH^K;PaRq%5 zSPzVyV8f@7;Ogmiw}pZ{6p^ce7XORTusodat+Y4Z#ev3epU~Ufgw$tV8vWs#QtVc$ zzza5K{}?CrZ`9^Xy7HBgKV|h%{0knm0F(SL7JYY(>b8m}dn8`Ki?OdqWxp#1yqrj{ zI6D;SVY}}5|Jp|tjWe_>rUJ|(Vh=4&XQy4_+R=s_=!R7l`|n1t^LlLjXFkB-pa<-B z>GK*5VweZR@mzaE?4z_$Q#7HkAb66IlR9`{>T1;CC2&YFLSAEg08^S2M61obPCaIQsDN+&A1$=vz8S?=WK)E z1U(y+7s7*F(nfhUnjf55I<}q`lHl>E%JQDr0ndle&A=Vbm>;E{{>S{^+=LT}wmtuT zWcH-8LeJLgE(>(jHXS8JI1Y9g0-G-b03yu-w7#mG)-Bm%_c11x`QNEj>745GuAk7d)nKw$~CqDuF4GR*N zTaU6Pede0!VZsP}iP_jZjq)m-$0)*OiYa;q)Ap2ppXT-}0^(?DL0O5nfR$oc7G|Wj zMG)X5aoI9OC!=*g(7rLULYGk}*2hEy6APhF@ioSo&K7uEP;r)pXJSPbgH2%{ALR5D zI1%`>a+h8%g9C@>1=zqEc09fp*^&L>OwY`#h)*{X5=c&|aCl69!b}hB2>s}F&O_;T z3V#P^v*$5V@Zni)_w?sS;QmjR}NFbWn@wX#&Oo4_yXKw1&c@-;WEj@1HLyl4#B z81uOcE73D5EZ5(>kCl`rL=|)_I~g%pX>7>|nzL4d5reH!gtAB6ZI}6-HgB*Qu#{)! zqG8JDY|kGitq;Sm81HuW-64qZ3nVN>{)LrO(Mt^*@ee};T7l1X@5Y4-*GI4pgkn|Q zG)z*Iz?#h8ACAvpfVCLB@APfohpLGBKr&g~$k1248+4+vD z{Vs-#me^|1Q|9%p@eOf_V_fZ+B?Ap~lM$;C-e|6dh z`Vx3#st-3NDg?cWCeTo-5##Wfsm-hnyMS-+IAh_>VP*E9WSaPGlD3>@G&DCFil} zaYyIbG@>Pp%E(M=1NKIv3E^!8+ejxI9pp}403}8->Iu_aKat0->(<<@PD}@%9kQ!% zjDpwhb6@bh6QLcu$=%#GHOPlBC!md*OYONfBxV>dHUG!Lm$ftvI7as{HZ6+(5UtYM zSL4K6iBV-Az8A$c3O<7Tl$_N?`TOAmMc2Zn3t(lvO51VIaL-tPjk?x)cq=7+P6rRQ zf)P)8En1$&2qwrlaTXrT7clw6Am_T>T*Lj{zN@Zrq%9C@;d6dpV(`4kQ6Var+``|~ z824zbB_tgEC8s6w;xZh4_NjwCJ@*B#J@X#@mA0C&Li@)7&dj)MG^_`PylM*MagtT~V?awo`O)0JEj<)c&rhR`DwIZVuLaYx7{k$`x3Hl4Oy$ z4=lPo!6|xnI*)mvc#OgpyPg~5eeVaVf|dmp`n(mJeWIm5za(otx?dO(`h0pLCaY1v9$;@h5adJ>kvNa*lqGvUhsHJJ*HOf7xwt9*io&{U~4oHL#GZ z=SJWIMEydy+=-p$Tafs!L*JaYHCZNPPh<5n)o_0GziPM^N2d4w z@2OLro=!PVlx{pHCyHvxts;BQDRh(MZn0A$wlgA^30s{Gl2|3gLMbW34w_-(o90sH zlIvo&F;R`#ja}Ns^YJ{t-=FaPemO`s_h{gW zkYsFHj⋙k*L*cwm3N*9qhTRCHzQbJDKv05L>8w&jkkxRrpLaUY|?er)*T)_=o33 z&u-1XHD49?y&tB1CJX=RT^F>_j|&BUxt1p<39`JV$REw<1IrT{2|EwAtE*@&QD6|) zf$t#P#wV{Qf7|gm=QkcKSDuE(4i~h91<`;vR+cU=uhBO8nzj*aJsu>xi%s8LUtGv{ zwjWvS@qCui(rKh=xs1`Vm@)@RieM^4X)~F56xG+};B1gBJ?adQ(INJJH!z6edHaFe zks|nZp2vqWoWfZ7g^D!WZHPmGFnjDi9LO!q3ppB)7Le-L{)L@yo4iee7I+T1bu)wn zMZOY$r}_J?D%fqwwKL3r+)khWss@JYO7YtZUypeFRkAd(?SG`;=0b*^L1|C7uJ> zU4@H1VQfFfrmg#*XH+^AghV&q_1o40DmGd7Cp6E>chf(jeA!{q%eVIaTk>#u)Gv~d zu@Ot1GQ=6wV*^X$@S2tH=WaIy$p(BefP@kv$>T7kUDSj|&eaF>gG9tYkq{y&5vc}8 zsCGvNr4rP~FD0j(yvBH>eVk6#Y#>_|QCx@Rk$n=McSX*h73gub+a49{|G_jFh>M}3 zWiU0TxS0MIbJx59ZddRE09~}V&p*Wanjf>|xc9n$0ns5nVM*`#CL=|Bfg({oEJkX6 zEE|Pi=7E;?IPHHeFXLp4aC>-(4QyOz`8mHM6n6WWWN<1!6}qDTr^zqMsj15mmB|ie zl)=C@cAx-BTVG2Rs~*^*Y1#_SY9&ch&QBLwNKi69yxqfyBN%%@O8KQvPJ9tcb;WRx zTk-^@J}Mu-X&*8bKGDddERydy{aUJf$NgUNYs26D96Pu%g+y}OCFj(SEGHtvr^4<- zVyI_S)^zQozAOFjF%anC82L4qxz6bftbv~x|Hf#>5yMCCHMX#sD; zg7IE%UogS>>{9RNp`O)!BR|ru7-Q&X{Oq1t?*@{2D%7_r##Qcxud^!|JKt|jmZShq zyj3}Oxmgz?tS!VhOgky~Yi`XH8hPxrCrn>C>2(diS%GHY_9xqp*q$aRPc9(B#%hIm z{-vsrDBvsGEym5{Ub7=QeW-lknzg;=4;4aN1p>V@$-hoK3U(xgM~0viOF1i5r?`kn zHRQmgPA`K-RtO00nVbzT_5*uE{?K&|J*{t~qYpb&!X20vYA}MDQX{7Fg+<+YHt5aw zI#}zr_g^^0iv#vpLCY)*2<21VbnB5pRG`n}yn3}F(pw8sD&Vlgs?`&n9X*k6 zx8e2A5TB=O&ugDehw0{ppin)!>x5!Fl3Z10baL*0+x0}_Qg_-hNedK){o7(=xUH}v z>I|lfNsUFuk!tHCN1HWWU@UOxLwl2eub)4o+X28vjuWujaHh)6)kG{yajv_*Ho)OM ziWmr@?N&NgH)P{imb_70O1GYDcK$Y_Gc^~SfMR<#x?6x|J~KfAtfA_$$!`#>d1J+) z+j#56hxXTkFSr)I@hvoz744)3eVE743h<-8Yy}$DHs*z@(Pp!J!iA&LX#(GTX1E+U z#@Zq_2|g)m`$Tj}2guA$G`kX3qCgZ}Qay`O;G{x8MygSHn{&bygddnqj#(qfsAk_B zWk^tuq99(VLWu0H4KZZh^1qcY8Qgm!;{bd_z#==z z=Il0lU&C{}i%2DOwFf<%W#zCup zlC|Aah6oQL-qo9;D6+^cm!(Ge+7JizK9EBkiOvdiYxXcmC7o8J-+Mm^o?*TFDt=;& z8yF4t4NurF%Ed(~P#rm9LEot`Nsq>AJNk1cdeg{wj})A+_LbG&k}$#s;`7cw6!u`1 zO^)K8dAn}`VkU?LM)H>QeyjH2ci!Dcx1PXWtZRJWn3>XoRzWtfM-T@bAo>~603Iui z9zLMNZii(%Te?paosvpNC+ix}K?C&jLZ8s6NuB6hJ-LrJwsOjzBq4MR2DdyPh>*+b z0s-i_f)U;lrGVwGKBxO2!?OKb#PSSml{_;|jTyiv=n{d?-$aieUJ1Ggo=nQ>_hfYH z>x)d+-EQ|VFhjVx&po@Mh70``7D1zDpxg>zZAtHU=m%{no7ph@FLmsap7{u{R8%At zQ%k_D6+k9BXDo6uz9eh_k3xA?s=eYrQj>?Z7+fDtuTB{uE_V@QQfoUmlw3624F>yJ zVXK{FGlx;h+smIIG(^t zqjeL=yUzzTjxXNJkp`$jcZQpJ)iuqz#5?Sko^8oAd`51IR7ifP`G@+1l08}ewhA$s zkoH}?L_p1QYP#dN_91wtHJ3~ znkl{Vu#f$a7W}?Zuems@qWuD%FD0cG0j;PE4yfu1+Es}0iF)m#>vjh^&psc@@w`m| z5C{iIQ%^{HEk;ahTWYTGnU5Az?3{hZYTK9eoMwfD^sCPz+->{n-7CnlNN|yc$vyfX zJNZ88H*{9)FD>SD|Jde0pR@qvJi|d#hgX2*0i!4_T$u!Z7Kz)<)F8ls)UpC7X{2nq zIJ=}5tKOSePGQ`y&yC4PerXLTa7fY8{Iq1Xpr}G=EfP*OVnckWnF?ex+k6^KV`(|V zW2jMm85w1Zdo2K|A-haRcM z8CHri%V_7IfyJ+{*#Ub+=1mxA7L&|_&8YHY$V{5DBEkv5N3P-6mzC}67;XqC%{4_Q zLuv=L4+qPX#}5B7iN5l8Xej<5?~*E0hYOUJ7(_I2bFYp?QeX5)+ojKGdG=96(OyX7 zbKQG-t*Jv`Q6SRL<9DTpIt&qj@(MtYRg7^?sLLq1f%Z|{NRYlvx&~zB(0VZ2+VVh& zSNr*+;}38xvYtk~C_W71Q#dk*uKxM)Zju5tQ=xz^#S3`iEWxRfu@2%o%jz$M*cM-) z8k6i(R`>Q4%36zUhF>@StfG^GPIqS^5)`2v4U$ux;aTS0TWncnw&>N)GjW=YYaE~i znI??l@Mckus%GlH4Clv-nk_)r=CzDy$`d%V3(2R6=jjbYTXcOtUD)}KxrVDEf+7Gy ze#jN?B{2D|{weQ)7OPS2F`Dkuxr6g+URx!Piz<&(U(xjk|1`*Qb5HtJ%9?heauh^0 zyG6l$m5tQYI>ISsht>V?bM$Bl{B4VI_SAXFdM$Zg-m}-wRN8SAN&@^yC&b}(TvMuV z5kKqfY-_OI$hMiVYykqf7)=J~gCQ5_cIm*nFsH=nuiRfG6vl6vic8BCbc*uX+zk&k zlBnx)-#%=9o_)bJ)R%tvY-wN6U}U6Oz0HXpGmxBOxjY+vZ_8X^J`*;U`{qjst_DSP zz$wab1bnM8exhiMCybEfu{un4fA|!&{)<$V%~~rRKMgG3#b8T9vvJ$x-DYlxkf3xZ z(XDNVr$*+>vy=r-ge~+8eC>b_`%`BVgsQxF;-bH&5h$2bI0KFC2b{6emGx`?oLQP9 zwqBSKPi6*{wY2$EciCzm9zHWI+Vn)^w^Mhdw7RHb2qyErfddH8I*pfRidNu9uMaHg zEikJ+-83eCOM1Io#TMH9EYMV*X__{Q{-{5b=DZ;IBjxMRo3EO~(RQ^6<=qHo-0G5p zj6@(EM|K_xV4EeKWUdSan9EoMB;4vDs&y6uCtMT z{{@H?gUuin`P06Degi7uonW%9Ch3*KTj(H0N)^iB%NjH26-c!WAA((F4?=yT3PQP% zQlXVy=*uXUx#!HWM_dEqH6;!`Iv}JN`FB1ZDBp#QZEX^-zIAPT-qleXH*lNlDEJAu zg2k!!IH!G>Y(0hkn&aR9v#N+ruIs9##cLp#n93!erw$cO1ypxD(?6R0g0c>@2l8eO2jY@OV#WA7v^i^z&keqFp^jpGPmNh+RBt zl-f?Vq}SH>u}^1^nOJ>sIPe5(I%s(XKn59dc1i{-*wPkMPu)-LsGvcyB`^+<<1?p3 zrXR8O5ykA@hBP-74w*}$=_7u`Dl4IND?f426Q6HKEr#_kusedF>;8cC9<53?PgC&s z6U46Y&4HuZXERMh@@hTj*}qbpn9am2RE0oI=-tMS$=%i4T}hq=G;z~dC|@`gVL zg@=u#Je>mHfPJ9&ZTRAYZ6wFEwE4rhg%}bB+^HM2!pcp1(~D!!Zx-+KZG9=mt?tGnnG}&8jP)-jzL|`-;vs-Ru}1Z!T=EaMpNiHrVP->atei z`jlrG@+-=ViIS2j)?dzs5npui5Pu!oj{RkM4u+z|fcFnZ&q4ro#dU#3%NAPu<%suz z_v+7+OutQ!3in_(`W)|9rrmW@CDOGTvn9L6Z*0bP^wZ zR%WfkPx7c^Rlk9aalJ1)mh|lWvoCv0sUVuh$7S=R8c_9sEFDS;-QiJ0bDiQH0XS_WdW)>|dR5Tn_1Mi2j1#FySV^QCm|*wh+IokX9#FuRMIHp8{brMMo-#C}&wz3{`m zqpR1T!pcJo#{z~*GWJzF(MxY!xL$(4zPo&-lbeKaS^L=80$a;yEllmmdU>?CYT^K> zE~#5VEwXc`> z9kDcP_%^02Ek1u$eetb-p6n3pCKG$VdKK;Plp9-846pWnYv|I&v*(Yxq3uiL1@(td zK=EfnVsv&n6@89-V&hcKUvpPdXTnQcTz7x__)tj-QNlw*^+p#X;k5M~2J;xR z&70W$%DyjL`$DtwIo8cf(wCm>gzJ0p)|O`w5Rm$=KJoZ>Z_1r4wT(9-8`2J3yCdGS z-_PT1LalBdh{2K+>^Nph~!g>gVEd~Y)ITPoBbw5&w0T!Xiff!8cae9R@jE!-}u7@_yN5h z9H2t25zqt)WRQ6bQC3su+v#wZ-+}Uk=Ulpw- zzf$G|VnY0e_s^4Gbm#Db#|vIOED_OSctxk4o;zu>_Iqt#;O-|m;ANsyWE$JSs#~i9 Y>?n_J9O}Do!Mr(kXZI6I&C=+90Is@cWdHyG diff --git a/docarray/typing/url/image_url.py b/docarray/typing/url/image_url.py index 25f07ed4707..4c25e68094e 100644 --- a/docarray/typing/url/image_url.py +++ b/docarray/typing/url/image_url.py @@ -49,34 +49,35 @@ def load( """ Load the data from the url into a numpy.ndarray image tensor - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from docarray import BaseDoc + from docarray.typing import ImageUrl + import numpy as np - from docarray import BaseDoc - from docarray.typing import ImageUrl - import numpy as np + class MyDoc(BaseDoc): + img_url: ImageUrl - class MyDoc(BaseDoc): - img_url: ImageUrl + doc = MyDoc( + img_url="https://upload.wikimedia.org/wikipedia/commons/8/80/" + "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg" + ) - doc = MyDoc( - img_url="https://upload.wikimedia.org/wikipedia/commons/8/80/" - "Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg" - ) - - img_tensor = doc.img_url.load() - assert isinstance(img_tensor, np.ndarray) + img_tensor = doc.img_url.load() + assert isinstance(img_tensor, np.ndarray) - img_tensor = doc.img_url.load(height=224, width=224) - assert img_tensor.shape == (224, 224, 3) + img_tensor = doc.img_url.load(height=224, width=224) + assert img_tensor.shape == (224, 224, 3) - layout = ('C', 'W', 'H') - img_tensor = doc.img_url.load(height=100, width=200, axis_layout=layout) - assert img_tensor.shape == (3, 200, 100) + layout = ('C', 'W', 'H') + img_tensor = doc.img_url.load(height=100, width=200, axis_layout=layout) + assert img_tensor.shape == (3, 200, 100) + ``` + --- :param width: width of the image tensor. :param height: height of the image tensor. diff --git a/docarray/typing/url/text_url.py b/docarray/typing/url/text_url.py index 049de511d1f..1ebfad1c511 100644 --- a/docarray/typing/url/text_url.py +++ b/docarray/typing/url/text_url.py @@ -43,31 +43,26 @@ def load(self, charset: str = 'utf-8', timeout: Optional[float] = None) -> str: """ Load the text file into a string. - EXAMPLE USAGE - .. code-block:: python + --- - from docarray import BaseDoc - from docarray.typing import TextUrl + ```python + from docarray import BaseDoc + from docarray.typing import TextUrl - class MyDoc(BaseDoc): - remote_url: TextUrl - local_url: TextUrl + class MyDoc(BaseDoc): + remote_url: TextUrl - doc = MyDoc( - remote_url='https://de.wikipedia.org/wiki/Brixen', - local_url='home/username/my_file.txt', - ) + doc = MyDoc( + remote_url='https://de.wikipedia.org/wiki/Brixen', + ) - remote_txt = doc.remote_url.load() - print(remote_txt) - # prints: ```\n ...``` + remote_txt = doc.remote_url.load() + ``` - local_txt = doc.local_url.load() - print(local_txt) - # prints content of my_file.txt + --- :param timeout: timeout (sec) for urlopen network request. diff --git a/docarray/typing/url/url_3d/mesh_url.py b/docarray/typing/url/url_3d/mesh_url.py index ce66cd3ad0b..2f235b6fb01 100644 --- a/docarray/typing/url/url_3d/mesh_url.py +++ b/docarray/typing/url/url_3d/mesh_url.py @@ -26,33 +26,34 @@ def load( trimesh_args: Optional[Dict[str, Any]] = None, ) -> 'VerticesAndFaces': """ - Load the data from the url into a VerticesAndFaces object containing - vertices and faces information. + Load the data from the url into a VerticesAndFaces object containing + vertices and faces information. - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from docarray import BaseDoc + import numpy as np - from docarray import BaseDoc - import numpy as np + from docarray.typing import Mesh3DUrl, NdArray - from docarray.typing import Mesh3DUrl, NdArray + class MyDoc(BaseDoc): + mesh_url: Mesh3DUrl - class MyDoc(BaseDoc): - mesh_url: Mesh3DUrl + doc = MyDoc(mesh_url="toydata/tetrahedron.obj") - doc = MyDoc(mesh_url="toydata/tetrahedron.obj") + tensors = doc.mesh_url.load() + assert isinstance(tensors.vertices, NdArray) + assert isinstance(tensors.faces, NdArray) + ``` - tensors = doc.mesh_url.load() - assert isinstance(tensors.vertices, NdArray) - assert isinstance(tensors.faces, NdArray) - - :param skip_materials: Skip materials if True, else skip. - :param trimesh_args: dictionary of additional arguments for `trimesh.load()` - or `trimesh.load_remote()`. - :return: VerticesAndFaces object containing vertices and faces information. + --- + :param skip_materials: Skip materials if True, else skip. + :param trimesh_args: dictionary of additional arguments for `trimesh.load()` + or `trimesh.load_remote()`. + :return: VerticesAndFaces object containing vertices and faces information. """ from docarray.documents.mesh.vertices_and_faces import VerticesAndFaces diff --git a/docarray/typing/url/url_3d/point_cloud_url.py b/docarray/typing/url/url_3d/point_cloud_url.py index 798c6a05aec..ac077795a4b 100644 --- a/docarray/typing/url/url_3d/point_cloud_url.py +++ b/docarray/typing/url/url_3d/point_cloud_url.py @@ -31,26 +31,29 @@ def load( """ Load the data from the url into an NdArray containing point cloud information. - EXAMPLE USAGE - .. code-block:: python + --- - import numpy as np - from docarray import BaseDoc + ```python + import numpy as np + from docarray import BaseDoc + + from docarray.typing import PointCloud3DUrl - from docarray.typing import PointCloud3DUrl + class MyDoc(BaseDoc): + point_cloud_url: PointCloud3DUrl - class MyDoc(BaseDoc): - point_cloud_url: PointCloud3DUrl + doc = MyDoc(point_cloud_url="toydata/tetrahedron.obj") - doc = MyDoc(point_cloud_url="toydata/tetrahedron.obj") + # point_cloud = doc.point_cloud_url.load(samples=100) - point_cloud = doc.point_cloud_url.load(samples=100) + # assert isinstance(point_cloud, np.ndarray) + # assert point_cloud.shape == (100, 3) + ``` - assert isinstance(point_cloud, np.ndarray) - assert point_cloud.shape == (100, 3) + --- :param samples: number of points to sample from the mesh :param multiple_geometries: if False, store point cloud in 2D np.ndarray. diff --git a/docarray/typing/url/video_url.py b/docarray/typing/url/video_url.py index 1ff5e96c9c4..e6b35642f5e 100644 --- a/docarray/typing/url/video_url.py +++ b/docarray/typing/url/video_url.py @@ -43,57 +43,61 @@ def load(self: T, **kwargs) -> VideoLoadResult: Load the data from the url into a named Tuple of VideoNdArray, AudioNdArray and NdArray. - :param kwargs: supports all keyword arguments that are being supported by - av.open() as described in: - https://pyav.org/docs/stable/api/_globals.html?highlight=open#av.open + --- - :return: AudioNdArray representing the audio content, VideoNdArray representing - the images of the video, NdArray of the key frame indices. + ```python + from typing import Optional + from docarray import BaseDoc - EXAMPLE USAGE + from docarray.typing import VideoUrl, VideoNdArray, AudioNdArray, NdArray - .. code-block:: python - from typing import Optional + class MyDoc(BaseDoc): + video_url: VideoUrl + video: Optional[VideoNdArray] + audio: Optional[AudioNdArray] + key_frame_indices: Optional[NdArray] - from docarray import BaseDoc - from docarray.typing import VideoUrl, VideoNdArray, AudioNdArray, NdArray + doc = MyDoc( + video_url='https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true' + ) + doc.video, doc.audio, doc.key_frame_indices = doc.video_url.load() + assert isinstance(doc.video, VideoNdArray) + assert isinstance(doc.audio, AudioNdArray) + assert isinstance(doc.key_frame_indices, NdArray) + ``` - class MyDoc(BaseDoc): - video_url: VideoUrl - video: Optional[VideoNdArray] - audio: Optional[AudioNdArray] - key_frame_indices: Optional[NdArray] + --- + You can load only the key frames (or video, audio respectively): - doc = MyDoc( - video_url='https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true' - ) - doc.video, doc.audio, doc.key_frame_indices = doc.video_url.load() + --- - assert isinstance(doc.video, VideoNdArray) - assert isinstance(doc.audio, AudioNdArray) - assert isinstance(doc.key_frame_indices, NdArray) + ```python + from pydantic import parse_obj_as - You can load only the key frames (or video, audio respectively): + from docarray.typing import NdArray, VideoUrl - .. code-block:: python - from pydantic import parse_obj_as + url = parse_obj_as( + VideoUrl, + 'https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true', + ) + key_frame_indices = url.load().key_frame_indices + assert isinstance(key_frame_indices, NdArray) + ``` - from docarray.typing import NdArray, VideoUrl + --- + :param kwargs: supports all keyword arguments that are being supported by + av.open() as described in: + https://pyav.org/docs/stable/api/_globals.html?highlight=open#av.open - url = parse_obj_as( - VideoUrl, - 'https://github.com/docarray/docarray/blob/feat-rewrite-v2/tests/toydata/mov_bbb.mp4?raw=true', - ) - key_frame_indices = url.load().key_frame_indices - assert isinstance(key_frame_indices, NdArray) - + :return: AudioNdArray representing the audio content, VideoNdArray representing + the images of the video, NdArray of the key frame indices. """ from docarray.typing.bytes.video_bytes import VideoBytes diff --git a/docs/api_references/typing/bytes.md b/docs/api_references/typing/bytes.md new file mode 100644 index 00000000000..a263581af0d --- /dev/null +++ b/docs/api_references/typing/bytes.md @@ -0,0 +1,6 @@ +# Bytes + +::: docarray.typing.bytes + + + diff --git a/docs/api_references/typing/id.md b/docs/api_references/typing/id.md new file mode 100644 index 00000000000..6dbfbd39dea --- /dev/null +++ b/docs/api_references/typing/id.md @@ -0,0 +1,6 @@ +# Id + +::: docarray.typing.id + + + diff --git a/docs/api_references/typing/tensor.md b/docs/api_references/typing/tensor.md new file mode 100644 index 00000000000..f8040705f32 --- /dev/null +++ b/docs/api_references/typing/tensor.md @@ -0,0 +1,3 @@ +# Tensor + +::: docarray.typing.tensor diff --git a/docs/api_references/typing/typing.md b/docs/api_references/typing/typing.md deleted file mode 100644 index fa29d35750c..00000000000 --- a/docs/api_references/typing/typing.md +++ /dev/null @@ -1,4 +0,0 @@ -# Typing - -::: docarray.typing - diff --git a/docs/api_references/typing/url.md b/docs/api_references/typing/url.md new file mode 100644 index 00000000000..f58c3b6c722 --- /dev/null +++ b/docs/api_references/typing/url.md @@ -0,0 +1,6 @@ +# Url + +::: docarray.typing.url + + + diff --git a/file_1.mp4 b/file_1.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..15b58e871afd89ea39501de2fd6f2618a77bd9fe GIT binary patch literal 8232 zcmeHMZ)hCV7QdTmW372rOSDz$W2EuVhh#Im*`#e?uxnadLBR@Ikq67&nY-CBGdq)+ zNxJciQHxgm@;*rArO#Ar_4%}-RQk$8eTh)vJrNO9w4&0|So~9JDQLPDUC+65?`CJ( z!t;ZEkR38}&%b-``Q0<;p4?eNh*}CKJS%V=LfQznaA)bbk#W3Sh7hu(Ax)o^{LW;nSy4Wi!L@7TDhC#UA` z9s-@o46qq;y$PF@LN%*teLYz%n+0Jx486hL-fi2qrN=Fkxi+n&U4OV2$4!?*#|9hM z3oW-245|h#QbW(EJ}c>cs>zDBYmDfbL2Xb|X@%Mo0n@c=-=J2_WcpNx>19?`gRvq| zYg8|ofUfW?_@U56H+Yeo%yX4_T()Y|G= z(+GhuMjc%P7pQr^TVZ-O)1AqvB^rbUFBq{rw1@^6^$KpO6tGb5$*N)5haLeqZ@caY zEdx_nkOj77fG<*zQMmxED?XphupAnqLslhZzD>azzZAw0M7!Fk|coRTdrk(|wn$$x$#X`}d0ZwB1j04=ptl@H41fA=#N@3XbKpP1? z=sLnCU~E0xuL-i?SQR~+Qv-ulm@yXWISt+PDXy*0f--dXjY0!A>U~3}MPGG_u)qkb zXn{Qf)LhpX;mLs2ZA7Eo}N=J526Qe3iN=n=_tGy7-ljF zFLRdd*iNpR`FZ|>o8H}jVyf%Fu@~m|OkZCZ8oMsFYv-X;N544B-aD}As?Sfa_DJgQ zM5I#Hel{Uy+2?uH$`50J&Tb5^NEEY~hgP17(xQYY;n zqB7OC?)ZugSI&@&KjExWi&(A&TjM@%`;oLA*o(G@r7dUMQM)`pRlfCweEDy;E$M%N zd;c0pJHeOdw@+=q^06moAKuDQE&7tnm0CjHhV~u#=|?tR-hK59A>(fWkLbTLw=93_ zr8*(6KFB3Xts&ty;N^2+{_#3Vy$yOq^v%`{D`!aClbls*5i7LQ|F{p97lxk&Z3h~U z@5Sj}o#Z@NM?BX9sC@v+UI2w5K~OIq0#N^30EL0+%qz8ooQDTMy|eL(IH(hQx7-;A z)sS#dm!zh@4&#oVd~!p~QNeRZ!Tj3&2a|YW{!*elDGsf zabfXl+UI4wc#%3!ekJ5GUXLVkL8(gY^9_l_1;s1oD3mLXOFAleY2K<|{GW|h{o?;` zwCb14FWlai!GB<_`X%=HR{e^~T~yEHKHsWeeBWr*FS&1EnZ(U@v;eR)1`bx$Yq;Iis0YE6=Oi0;MXr$o{wrH zr2R+uZaf0Yzl0-vPNm6Ej0W>2uf#m(o%aZI!kdsTW`+T@tC$_Y?;-qx5pdi^!smA1 zF)a!j)iIOUH4y)o+;xce>nWSI&nbA|?;euI_HO(gWHYUpHba~CYv6--35d>dhY#B9 zc9$9HR+;dX9XrE?`i?Q*RwH`x=RsJsfqpp%0~rvsc?y5*ffldkF(b(lfV~+=yzx5M zB9bpWSJvU%D(Qn!hf*f#tRE#ShN0bec<9mcdO{Qej}GPSrED@q7MH(0vA_GLGiR`S zCw%{pR`k-RflX6*tPDcF+CcKD!Ep#Q*a8TQ0$%76+{%r{f~^tBAR=71RJ4*u7W?qA zpl#`+^=Pe6v<~=s{E^;|C5Q)JcK*Jvg51IUz)R!~fxJRGwHI{vb4~OBxk7>Lhr7G! zz6I!<;CrE!Jkl-xw_Oj|`wSP>883_6!lg?<`FJvAx)o^{LW;nSy4Wi!L@7TDhC#UA` z9s-@o46qq;y$PF@LN%*teLYz%n+0Jx486hL-fi2qrN=Fkxi+n&U4OV2$4!?*#|9hM z3oW-245|h#QbW(EJ}c>cs>zDBYmDfbL2Xb|X@%Mo0n@c=-=J2_WcpNx>19?`gRvq| zYg8|ofUfW?_@U56H+Yeo%yX4_T()Y|G= z(+GhuMjc%P7pQr^TVZ-O)1AqvB^rbUFBq{rw1@^6^$KpO6tGb5$*N)5haLeqZ@caY zEdx_nkOj77fG<*zQMmxED?XphupAnqLslhZzD>azzZAw0M7!Fk|coRTdrk(|wn$$x$#X`}d0ZwB1j04=ptl@H41fA=#N@3XbKpP1? z=sLnCU~E0xuL-i?SQR~+Qv-ulm@yXWISt+PDXy*0f--dXjY0!A>U~3}MPGG_u)qkb zXn{Qf)LhpX;mLs2ZA7Eo}N=J526Qe3iN=n=_tGy7-ljF zFLRdd*iNpR`FZ|>o8H}jVyf%Fu@~m|OkZCZ8oMsFYv-X;N544B-aD}As?Sfa_DJgQ zM5I#Hel{Uy+2?uH$`50J&Tb5^NEEY~hgP17(xQYY;n zqB7OC?)ZugSI&@&KjExWi&(A&TjM@%`;oLA*o(G@r7dUMQM)`pRlfCweEDy;E$M%N zd;c0pJHeOdw@+=q^06moAKuDQE&7tnm0CjHhV~u#=|?tR-hK59A>(fWkLbTLw=93_ zr8*(6KFB3Xts&ty;N^2+{_#3Vy$yOq^v%`{D`!aClbls*5i7LQ|F{p97lxk&Z3h~U z@5Sj}o#Z@NM?BX9sC@v+UI2w5K~OIq0#N^30EL0+%qz8ooQDTMy|eL(IH(hQx7-;A z)sS#dm!zh@4&#oVd~!p~QNeRZ!Tj3&2a|YW{!*elDGsf zabfXl+UI4wc#%3!ekJ5GUXLVkL8(gY^9_l_1;s1oD3mLXOFAleY2K<|{GW|h{o?;` zwCb14FWlai!GB<_`X%=HR{e^~T~yEHKHsWeeBWr*FS&1EnZ(U@v;eR)1`bx$Yq;Iis0YE6=Oi0;MXr$o{wrH zr2R+uZaf0Yzl0-vPNm6Ej0W>2uf#m(o%aZI!kdsTW`+T@tC$_Y?;-qx5pdi^!smA1 zF)a!j)iIOUH4y)o+;xce>nWSI&nbA|?;euI_HO(gWHYUpHba~CYv6--35d>dhY#B9 zc9$9HR+;dX9XrE?`i?Q*RwH`x=RsJsfqpp%0~rvsc?y5*ffldkF(b(lfV~+=yzx5M zB9bpWSJvU%D(Qn!hf*f#tRE#ShN0bec<9mcdO{Qej}GPSrED@q7MH(0vA_GLGiR`S zCw%{pR`k-RflX6*tPDcF+CcKD!Ep#Q*a8TQ0$%76+{%r{f~^tBAR=71RJ4*u7W?qA zpl#`+^=Pe6v<~=s{E^;|C5Q)JcK*Jvg51IUz)R!~fxJRGwHI{vb4~OBxk7>Lhr7G! zz6I!<;CrE!Jkl-xw_Oj|`wSP>883_6!lg?<`FJv Date: Wed, 29 Mar 2023 11:22:45 +0200 Subject: [PATCH 46/66] fix: fix for doc_string test Signed-off-by: anna-charlotte --- docarray/base_doc/__init__.py | 7 +- docarray/index/__init__.py | 9 +- docarray/store/__init__.py | 11 +- docarray/typing/__init__.py | 190 +++++++------------ docarray/typing/tensor/__init__.py | 54 +++--- docarray/typing/tensor/audio/__init__.py | 10 +- docarray/typing/tensor/embedding/__init__.py | 10 +- docarray/typing/tensor/image/__init__.py | 10 +- docarray/typing/tensor/video/__init__.py | 11 +- 9 files changed, 136 insertions(+), 176 deletions(-) diff --git a/docarray/base_doc/__init__.py b/docarray/base_doc/__init__.py index 489ed0015a0..04f8e2ee68a 100644 --- a/docarray/base_doc/__init__.py +++ b/docarray/base_doc/__init__.py @@ -1,16 +1,17 @@ from docarray.base_doc.any_doc import AnyDoc from docarray.base_doc.base_node import BaseNode from docarray.base_doc.doc import BaseDoc +from docarray.utils._internal.misc import import_library __all__ = ['AnyDoc', 'BaseDoc', 'BaseNode'] -from docarray.utils._internal.misc import import_library - def __getattr__(name: str): + if name not in __all__: + __all__.append(name) + if name == 'DocResponse': import_library('fastapi', raise_error=True) from docarray.base_doc.doc_response import DocResponse - __all__.extend(['DocResponse']) return DocResponse diff --git a/docarray/index/__init__.py b/docarray/index/__init__.py index 55a3e8dd318..8bdc8366e87 100644 --- a/docarray/index/__init__.py +++ b/docarray/index/__init__.py @@ -2,16 +2,19 @@ from docarray.utils._internal.misc import import_library -__all__ = [] - if TYPE_CHECKING: from docarray.index.backends.hnswlib import HnswDocumentIndex # noqa: F401 +__all__ = [] + def __getattr__(name: str): + + if name not in __all__: + __all__.append(name) + if name == 'HnswDocumentIndex': import_library('hnswlib', raise_error=True) from docarray.index.backends.hnswlib import HnswDocumentIndex # noqa - __all__.append('HnswDocumentIndex') return HnswDocumentIndex diff --git a/docarray/store/__init__.py b/docarray/store/__init__.py index c1174042e41..1d59906af2b 100644 --- a/docarray/store/__init__.py +++ b/docarray/store/__init__.py @@ -1,23 +1,23 @@ from typing import TYPE_CHECKING from docarray.store.file import FileDocStore - -__all__ = ['FileDocStore'] - - from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from docarray.store.jac import JACDocStore # noqa: F401 from docarray.store.s3 import S3DocStore # noqa: F401 +__all__ = ['FileDocStore'] + def __getattr__(name: str): + if name not in __all__: + __all__.append(name) + if name == 'JACDocStore': import_library('hubble', raise_error=True) from docarray.store.jac import JACDocStore # noqa - __all__.append('JACDocStore') return JACDocStore elif name == 'S3DocStore': @@ -26,5 +26,4 @@ def __getattr__(name: str): import_library('boto3', raise_error=True) from docarray.store.s3 import S3DocStore # noqa - __all__.append('S3DocStore') return S3DocStore diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index 5cd9a6367ad..bcf5e2233c8 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -1,6 +1,5 @@ from typing_extensions import TYPE_CHECKING -from docarray.typing.bytes import ImageBytes from docarray.typing.bytes import AudioBytes, ImageBytes, VideoBytes from docarray.typing.id import ID from docarray.typing.tensor import ImageNdArray, ImageTensor @@ -18,6 +17,19 @@ TextUrl, VideoUrl, ) +from docarray.utils._internal.misc import import_library + +if TYPE_CHECKING: + from docarray.typing.tensor import TensorFlowTensor # noqa: F401 + from docarray.typing.tensor import TorchEmbedding, TorchTensor # noqa: F401 + from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F401 + from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 + from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 + from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.video import VideoTorchTensor # noqa: F401 + __all__ = [ 'NdArray', @@ -34,7 +46,6 @@ 'AnyUrl', 'ID', 'AnyTensor', - 'ImageBytes', 'ImageTensor', 'ImageNdArray', 'ImageBytes', @@ -43,122 +54,63 @@ ] -if TYPE_CHECKING: - from docarray.typing.tensor import TensorFlowTensor # noqa: F401 - from docarray.typing.tensor import TorchEmbedding, TorchTensor # noqa: F401 - from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F401 - from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 - from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 - from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.video import VideoTorchTensor # noqa: F401 +def __getattr__(name: str): + # if 'Torch' in name: + # import_library('torch', raise_error=True) + # elif 'TensorFlow' in name: + # import_library('tensorflow', raise_error=True) + # + # from docarray.typing import tensor + # + # T = getattr(tensor, name) + # if name not in __all__: + # __all__.append(name) + # return T -from docarray.utils._internal.misc import is_tf_available, is_torch_available + if name not in __all__: + __all__.append(name) + if 'Torch' in name: + import_library('torch', raise_error=True) -torch_available = is_torch_available() -if torch_available: - from docarray.typing.tensor import TorchEmbedding, TorchTensor # noqa: F401 - from docarray.typing.tensor.audio.audio_torch_tensor import AudioTorchTensor # noqa - from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 - from docarray.typing.tensor.video.video_torch_tensor import VideoTorchTensor # noqa - - __all__.extend( - [ - 'AudioTorchTensor', - 'TorchEmbedding', - 'TorchTensor', - 'VideoTorchTensor', - 'ImageTorchTensor', - ] - ) - -tf_available = is_tf_available() -if tf_available: - from docarray.typing.tensor import TensorFlowTensor # noqa: F401 - from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 - from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa - - __all__.extend( - [ - 'TensorFlowTensor', - 'TensorFlowEmbedding', - 'AudioTensorFlowTensor', - 'ImageTensorFlowTensor', - 'VideoTensorFlowTensor', - ] - ) - -# -# def __getattr__(name: str): -# -# torch_tensors = [ -# 'AudioTorchTensor', -# 'TorchEmbedding', -# 'TorchTensor', -# 'VideoTorchTensor', -# 'ImageTorchTensor', -# ] -# -# tf_tensors = [ -# 'TensorFlowTensor', -# 'TensorFlowEmbedding', -# 'AudioTensorFlowTensor', -# 'ImageTensorFlowTensor', -# 'VideoTensorFlowTensor', -# ] -# -# if name in torch_tensors: -# import_library('torch', raise_error=True) -# -# from docarray.typing.tensor import TorchEmbedding, TorchTensor # noqa: F811 -# from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F811 -# from docarray.typing.tensor.image import ImageTorchTensor # noqa: F811 -# from docarray.typing.tensor.video import VideoTorchTensor # noqa: F811 -# -# # __all__.extend(torch_tensors) -# -# if name == 'TorchTensor': -# __all__.append('TorchTensor') -# return TorchTensor -# elif name == 'TorchEmbedding': -# __all__.append('TorchEmbedding') -# return TorchEmbedding -# elif name == 'AudioTorchTensor': -# __all__.append('AudioTorchTensor') -# return AudioTorchTensor -# elif name == 'ImageTorchTensor': -# __all__.append('ImageTorchTensor') -# return ImageTorchTensor -# elif name == 'VideoTorchTensor': -# __all__.append('VideoTorchTensor') -# return VideoTorchTensor -# -# elif name in tf_tensors: -# import_library('tensorflow', raise_error=True) -# -# from docarray.typing.tensor import TensorFlowTensor # noqa -# from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa -# from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa -# from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa -# from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa -# -# # __all__.extend(torch_tensors) -# -# if name == 'TensorFlowTensor': -# __all__.append('TensorFlowTensor') -# return TensorFlowTensor -# elif name == 'TensorFlowEmbedding': -# __all__.append('TensorFlowEmbedding') -# return TensorFlowEmbedding -# elif name == 'AudioTensorFlowTensor': -# __all__.append('AudioTensorFlowTensor') -# return AudioTensorFlowTensor -# elif name == 'ImageTensorFlowTensor': -# __all__.append('ImageTensorFlowTensor') -# return ImageTensorFlowTensor -# elif name == 'VideoTensorFlowTensor': -# __all__.append('VideoTensorFlowTensor') -# return VideoTensorFlowTensor + if name == 'TorchTensor': + from docarray.typing.tensor import TorchTensor # noqa: F811 + + return TorchTensor + elif name == 'TorchEmbedding': + from docarray.typing.tensor import TorchEmbedding # noqa: F811 + + return TorchEmbedding + elif name == 'AudioTorchTensor': + from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F811 + + return AudioTorchTensor + elif name == 'ImageTorchTensor': + from docarray.typing.tensor.image import ImageTorchTensor # noqa: F811 + + return ImageTorchTensor + elif name == 'VideoTorchTensor': + from docarray.typing.tensor.video import VideoTorchTensor # noqa: F811 + + return VideoTorchTensor + elif 'TensorFlow' in name: + import_library('tensorflow', raise_error=True) + if name == 'TensorFlowTensor': + from docarray.typing.tensor import TensorFlowTensor # noqa + + return TensorFlowTensor + elif name == 'TensorFlowEmbedding': + from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa + + return TensorFlowEmbedding + elif name == 'AudioTensorFlowTensor': + from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa + + return AudioTensorFlowTensor + elif name == 'ImageTensorFlowTensor': + from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa + + return ImageTensorFlowTensor + elif name == 'VideoTensorFlowTensor': + from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa + + return VideoTensorFlowTensor diff --git a/docarray/typing/tensor/__init__.py b/docarray/typing/tensor/__init__.py index 7bd6caa9af0..483676fe5f3 100644 --- a/docarray/typing/tensor/__init__.py +++ b/docarray/typing/tensor/__init__.py @@ -6,6 +6,16 @@ from docarray.typing.tensor.ndarray import NdArray from docarray.typing.tensor.tensor import AnyTensor from docarray.typing.tensor.video import VideoNdArray +from docarray.utils._internal.misc import import_library + +if TYPE_CHECKING: + from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 + from docarray.typing.tensor.embedding import TorchEmbedding # noqa: F401 + from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 + from docarray.typing.tensor.tensorflow_tensor import TensorFlowTensor # noqa: F401 + from docarray.typing.tensor.torch_tensor import TorchTensor # noqa: F401 + from docarray.typing.tensor.video import VideoTorchTensor # noqa: F401 __all__ = [ 'NdArray', @@ -18,50 +28,44 @@ 'VideoNdArray', ] -from docarray.utils._internal.misc import import_library - -if TYPE_CHECKING: - from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 - from docarray.typing.tensor.embedding import TorchEmbedding # noqa: F401 - from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 - from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 - from docarray.typing.tensor.tensorflow_tensor import TensorFlowTensor # noqa: F401 - from docarray.typing.tensor.torch_tensor import TorchTensor # noqa: F401 - from docarray.typing.tensor.video import VideoTorchTensor # noqa: F401 -torch_tensors = ['TorchEmbedding', 'ImageTorchTensor', 'TorchTensor'] -tf_tensors = ['TensorFlowEmbedding', 'TensorFlowTensor', 'ImageTensorFlowTensor'] +def __getattr__(name: str): + if name not in __all__: + __all__.append(name) + torch_tensors = ['TorchEmbedding', 'ImageTorchTensor', 'TorchTensor'] + tf_tensors = ['TensorFlowEmbedding', 'TensorFlowTensor', 'ImageTensorFlowTensor'] -def __getattr__(name: str): if name in torch_tensors: import_library('torch', raise_error=True) - from docarray.typing.tensor.embedding import TorchEmbedding # noqa - from docarray.typing.tensor.image import ImageTorchTensor # noqa - from docarray.typing.tensor.torch_tensor import TorchTensor # noqa - - __all__.extend(torch_tensors) - if name == 'TorchTensor': + from docarray.typing.tensor.torch_tensor import TorchTensor # noqa + return TorchTensor elif name == 'TorchEmbedding': + from docarray.typing.tensor.embedding import TorchEmbedding # noqa + return TorchEmbedding elif name == 'ImageTorchTensor': + from docarray.typing.tensor.image import ImageTorchTensor # noqa + return ImageTorchTensor elif name in tf_tensors: import_library('tensorflow', raise_error=True) - from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa - from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa - from docarray.typing.tensor.tensorflow_tensor import TensorFlowTensor # noqa - - __all__.extend(tf_tensors) - if name == 'TensorFlowTensor': + from docarray.typing.tensor.tensorflow_tensor import ( # noqa + TensorFlowTensor, + ) + return TensorFlowTensor elif name == 'TensorFlowEmbedding': + from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa + return TensorFlowEmbedding elif name == 'ImageTensorFlowTensor': + from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa + return ImageTensorFlowTensor diff --git a/docarray/typing/tensor/audio/__init__.py b/docarray/typing/tensor/audio/__init__.py index 30973a7ad12..384fb90bb3d 100644 --- a/docarray/typing/tensor/audio/__init__.py +++ b/docarray/typing/tensor/audio/__init__.py @@ -2,9 +2,6 @@ from docarray.typing.tensor.audio.audio_ndarray import AudioNdArray from docarray.typing.tensor.audio.audio_tensor import AudioTensor - -__all__ = ['AudioNdArray', 'AudioTensor'] - from docarray.utils._internal.misc import import_library if TYPE_CHECKING: @@ -13,15 +10,19 @@ ) from docarray.typing.tensor.audio.audio_torch_tensor import AudioTorchTensor # noqa +__all__ = ['AudioNdArray', 'AudioTensor'] + def __getattr__(name: str): + if name not in __all__: + __all__.append(name) + if name == 'AudioTorchTensor': import_library('torch', raise_error=True) from docarray.typing.tensor.audio.audio_torch_tensor import ( # noqa AudioTorchTensor, ) - __all__.append('AudioTorchTensor') return AudioTorchTensor elif name == 'AudioTensorFlowTensor': @@ -31,5 +32,4 @@ def __getattr__(name: str): AudioTensorFlowTensor, ) - __all__.append('AudioTensorFlowTensor') return AudioTensorFlowTensor diff --git a/docarray/typing/tensor/embedding/__init__.py b/docarray/typing/tensor/embedding/__init__.py index 42814851999..fe1aca442a8 100644 --- a/docarray/typing/tensor/embedding/__init__.py +++ b/docarray/typing/tensor/embedding/__init__.py @@ -2,22 +2,23 @@ from docarray.typing.tensor.embedding.embedding import AnyEmbedding from docarray.typing.tensor.embedding.ndarray import NdArrayEmbedding - -__all__ = ['NdArrayEmbedding', 'AnyEmbedding'] - from docarray.utils._internal.misc import import_library if TYPE_CHECKING: from docarray.typing.tensor.embedding.tensorflow import TensorFlowEmbedding # noqa from docarray.typing.tensor.embedding.torch import TorchEmbedding # noqa +__all__ = ['NdArrayEmbedding', 'AnyEmbedding'] + def __getattr__(name: str): + if name not in __all__: + __all__.append(name) + if name == 'TorchEmbedding': import_library('torch', raise_error=True) from docarray.typing.tensor.embedding.torch import TorchEmbedding # noqa - __all__.append('TorchEmbedding') return TorchEmbedding elif name == 'TensorFlowEmbedding': @@ -27,5 +28,4 @@ def __getattr__(name: str): TensorFlowEmbedding, ) - __all__.append('TensorFlowEmbedding') return TensorFlowEmbedding diff --git a/docarray/typing/tensor/image/__init__.py b/docarray/typing/tensor/image/__init__.py index 1c199a5ff81..1d8d7108b2a 100644 --- a/docarray/typing/tensor/image/__init__.py +++ b/docarray/typing/tensor/image/__init__.py @@ -2,9 +2,6 @@ from docarray.typing.tensor.image.image_ndarray import ImageNdArray from docarray.typing.tensor.image.image_tensor import ImageTensor - -__all__ = ['ImageNdArray', 'ImageTensor'] - from docarray.utils._internal.misc import import_library if TYPE_CHECKING: @@ -13,15 +10,19 @@ ) from docarray.typing.tensor.image.image_torch_tensor import ImageTorchTensor # noqa +__all__ = ['ImageNdArray', 'ImageTensor'] + def __getattr__(name: str): + if name not in __all__: + __all__.append(name) + if name == 'ImageTorchTensor': import_library('torch', raise_error=True) from docarray.typing.tensor.image.image_torch_tensor import ( # noqa ImageTorchTensor, ) - __all__.append('ImageTorchTensor') return ImageTorchTensor elif name == 'ImageTensorFlowTensor': @@ -31,5 +32,4 @@ def __getattr__(name: str): ImageTensorFlowTensor, ) - __all__.append('ImageTensorFlowTensor') return ImageTensorFlowTensor diff --git a/docarray/typing/tensor/video/__init__.py b/docarray/typing/tensor/video/__init__.py index 8262e30f357..210dfcf8f6a 100644 --- a/docarray/typing/tensor/video/__init__.py +++ b/docarray/typing/tensor/video/__init__.py @@ -2,9 +2,6 @@ from docarray.typing.tensor.video.video_ndarray import VideoNdArray from docarray.typing.tensor.video.video_tensor import VideoTensor - -__all__ = ['VideoNdArray', 'VideoTensor'] - from docarray.utils._internal.misc import import_library if TYPE_CHECKING: @@ -13,15 +10,20 @@ ) from docarray.typing.tensor.video.video_torch_tensor import VideoTorchTensor # noqa +__all__ = ['VideoNdArray', 'VideoTensor'] + def __getattr__(name: str): + + if name not in __all__: + __all__.append(name) + if name == 'VideoTorchTensor': import_library('torch', raise_error=True) from docarray.typing.tensor.video.video_torch_tensor import ( # noqa VideoTorchTensor, ) - __all__.append('VideoTorchTensor') return VideoTorchTensor elif name == 'VideoTensorFlowTensor': @@ -31,5 +33,4 @@ def __getattr__(name: str): VideoTensorFlowTensor, ) - __all__.append('VideoTensorFlowTensor') return VideoTensorFlowTensor From bf055c5ed149cd0f9abae7ff6ea23a2c1d95c998 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Wed, 29 Mar 2023 11:34:55 +0200 Subject: [PATCH 47/66] fix: try short version in typing init getattr Signed-off-by: anna-charlotte --- docarray/typing/__init__.py | 103 ++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 51 deletions(-) diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index bcf5e2233c8..4d691ded024 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -55,62 +55,63 @@ def __getattr__(name: str): - # if 'Torch' in name: - # import_library('torch', raise_error=True) - # elif 'TensorFlow' in name: - # import_library('tensorflow', raise_error=True) - # - # from docarray.typing import tensor - # - # T = getattr(tensor, name) - # if name not in __all__: - # __all__.append(name) - # return T - - if name not in __all__: - __all__.append(name) if 'Torch' in name: import_library('torch', raise_error=True) - - if name == 'TorchTensor': - from docarray.typing.tensor import TorchTensor # noqa: F811 - - return TorchTensor - elif name == 'TorchEmbedding': - from docarray.typing.tensor import TorchEmbedding # noqa: F811 - - return TorchEmbedding - elif name == 'AudioTorchTensor': - from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F811 - - return AudioTorchTensor - elif name == 'ImageTorchTensor': - from docarray.typing.tensor.image import ImageTorchTensor # noqa: F811 - - return ImageTorchTensor - elif name == 'VideoTorchTensor': - from docarray.typing.tensor.video import VideoTorchTensor # noqa: F811 - - return VideoTorchTensor elif 'TensorFlow' in name: import_library('tensorflow', raise_error=True) - if name == 'TensorFlowTensor': - from docarray.typing.tensor import TensorFlowTensor # noqa - return TensorFlowTensor - elif name == 'TensorFlowEmbedding': - from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa + from docarray.typing import tensor - return TensorFlowEmbedding - elif name == 'AudioTensorFlowTensor': - from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa - - return AudioTensorFlowTensor - elif name == 'ImageTensorFlowTensor': - from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa + T = getattr(tensor, name) + if name not in __all__: + __all__.append(name) - return ImageTensorFlowTensor - elif name == 'VideoTensorFlowTensor': - from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa + return T - return VideoTensorFlowTensor + # if name not in __all__: + # __all__.append(name) + # if 'Torch' in name: + # import_library('torch', raise_error=True) + # + # if name == 'TorchTensor': + # from docarray.typing.tensor import TorchTensor # noqa: F811 + # + # return TorchTensor + # elif name == 'TorchEmbedding': + # from docarray.typing.tensor import TorchEmbedding # noqa: F811 + # + # return TorchEmbedding + # elif name == 'AudioTorchTensor': + # from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F811 + # + # return AudioTorchTensor + # elif name == 'ImageTorchTensor': + # from docarray.typing.tensor.image import ImageTorchTensor # noqa: F811 + # + # return ImageTorchTensor + # elif name == 'VideoTorchTensor': + # from docarray.typing.tensor.video import VideoTorchTensor # noqa: F811 + # + # return VideoTorchTensor + # elif 'TensorFlow' in name: + # import_library('tensorflow', raise_error=True) + # if name == 'TensorFlowTensor': + # from docarray.typing.tensor import TensorFlowTensor # noqa + # + # return TensorFlowTensor + # elif name == 'TensorFlowEmbedding': + # from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa + # + # return TensorFlowEmbedding + # elif name == 'AudioTensorFlowTensor': + # from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa + # + # return AudioTensorFlowTensor + # elif name == 'ImageTensorFlowTensor': + # from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa + # + # return ImageTensorFlowTensor + # elif name == 'VideoTensorFlowTensor': + # from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa + # + # return VideoTensorFlowTensor From 80a2dab19a459aac50ddd24fca52bd5c15eb76f4 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Wed, 29 Mar 2023 13:57:50 +0200 Subject: [PATCH 48/66] fix: shorter version in getattr Signed-off-by: anna-charlotte --- docarray/base_doc/__init__.py | 6 +-- docarray/index/__init__.py | 7 ++- docarray/store/__init__.py | 17 +++--- docarray/typing/__init__.py | 55 ++----------------- docarray/typing/tensor/__init__.py | 56 ++++++++------------ docarray/typing/tensor/audio/__init__.py | 21 +++----- docarray/typing/tensor/embedding/__init__.py | 18 +++---- docarray/typing/tensor/image/__init__.py | 20 +++---- docarray/typing/tensor/video/__init__.py | 21 +++----- 9 files changed, 69 insertions(+), 152 deletions(-) diff --git a/docarray/base_doc/__init__.py b/docarray/base_doc/__init__.py index 04f8e2ee68a..e835671439e 100644 --- a/docarray/base_doc/__init__.py +++ b/docarray/base_doc/__init__.py @@ -7,11 +7,11 @@ def __getattr__(name: str): - if name not in __all__: - __all__.append(name) - if name == 'DocResponse': import_library('fastapi', raise_error=True) from docarray.base_doc.doc_response import DocResponse + if name not in __all__: + __all__.append(name) + return DocResponse diff --git a/docarray/index/__init__.py b/docarray/index/__init__.py index 8bdc8366e87..2ddef0931af 100644 --- a/docarray/index/__init__.py +++ b/docarray/index/__init__.py @@ -9,12 +9,11 @@ def __getattr__(name: str): - - if name not in __all__: - __all__.append(name) - if name == 'HnswDocumentIndex': import_library('hnswlib', raise_error=True) from docarray.index.backends.hnswlib import HnswDocumentIndex # noqa + if name not in __all__: + __all__.append(name) + return HnswDocumentIndex diff --git a/docarray/store/__init__.py b/docarray/store/__init__.py index 1d59906af2b..2c005ea1bfd 100644 --- a/docarray/store/__init__.py +++ b/docarray/store/__init__.py @@ -11,19 +11,18 @@ def __getattr__(name: str): - if name not in __all__: - __all__.append(name) - if name == 'JACDocStore': import_library('hubble', raise_error=True) - from docarray.store.jac import JACDocStore # noqa - - return JACDocStore - + import docarray.store.jac as lib elif name == 'S3DocStore': import_library('smart_open', raise_error=True) import_library('botocore', raise_error=True) import_library('boto3', raise_error=True) - from docarray.store.s3 import S3DocStore # noqa + import docarray.store.s3 as lib + + tensor_cls = getattr(lib, name) + + if name not in __all__: + __all__.append(name) - return S3DocStore + return tensor_cls diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index 4d691ded024..b00dcdf45d1 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -60,58 +60,11 @@ def __getattr__(name: str): elif 'TensorFlow' in name: import_library('tensorflow', raise_error=True) - from docarray.typing import tensor + import docarray.typing.tensor + + tensor_cls = getattr(docarray.typing.tensor, name) - T = getattr(tensor, name) if name not in __all__: __all__.append(name) - return T - - # if name not in __all__: - # __all__.append(name) - # if 'Torch' in name: - # import_library('torch', raise_error=True) - # - # if name == 'TorchTensor': - # from docarray.typing.tensor import TorchTensor # noqa: F811 - # - # return TorchTensor - # elif name == 'TorchEmbedding': - # from docarray.typing.tensor import TorchEmbedding # noqa: F811 - # - # return TorchEmbedding - # elif name == 'AudioTorchTensor': - # from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F811 - # - # return AudioTorchTensor - # elif name == 'ImageTorchTensor': - # from docarray.typing.tensor.image import ImageTorchTensor # noqa: F811 - # - # return ImageTorchTensor - # elif name == 'VideoTorchTensor': - # from docarray.typing.tensor.video import VideoTorchTensor # noqa: F811 - # - # return VideoTorchTensor - # elif 'TensorFlow' in name: - # import_library('tensorflow', raise_error=True) - # if name == 'TensorFlowTensor': - # from docarray.typing.tensor import TensorFlowTensor # noqa - # - # return TensorFlowTensor - # elif name == 'TensorFlowEmbedding': - # from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa - # - # return TensorFlowEmbedding - # elif name == 'AudioTensorFlowTensor': - # from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa - # - # return AudioTensorFlowTensor - # elif name == 'ImageTensorFlowTensor': - # from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa - # - # return ImageTensorFlowTensor - # elif name == 'VideoTensorFlowTensor': - # from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa - # - # return VideoTensorFlowTensor + return tensor_cls diff --git a/docarray/typing/tensor/__init__.py b/docarray/typing/tensor/__init__.py index 483676fe5f3..6377712bd14 100644 --- a/docarray/typing/tensor/__init__.py +++ b/docarray/typing/tensor/__init__.py @@ -9,12 +9,15 @@ from docarray.utils._internal.misc import import_library if TYPE_CHECKING: + from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 + from docarray.typing.tensor.audio import AudioTorchTensor # noqa: F401 from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa: F401 from docarray.typing.tensor.embedding import TorchEmbedding # noqa: F401 from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa: F401 from docarray.typing.tensor.image import ImageTorchTensor # noqa: F401 from docarray.typing.tensor.tensorflow_tensor import TensorFlowTensor # noqa: F401 from docarray.typing.tensor.torch_tensor import TorchTensor # noqa: F401 + from docarray.typing.tensor.video import VideoTensorFlowTensor # noqa: F401 from docarray.typing.tensor.video import VideoTorchTensor # noqa: F401 __all__ = [ @@ -30,42 +33,27 @@ def __getattr__(name: str): - - if name not in __all__: - __all__.append(name) - torch_tensors = ['TorchEmbedding', 'ImageTorchTensor', 'TorchTensor'] - tf_tensors = ['TensorFlowEmbedding', 'TensorFlowTensor', 'ImageTensorFlowTensor'] - - if name in torch_tensors: + if 'Torch' in name: import_library('torch', raise_error=True) - - if name == 'TorchTensor': - from docarray.typing.tensor.torch_tensor import TorchTensor # noqa - - return TorchTensor - elif name == 'TorchEmbedding': - from docarray.typing.tensor.embedding import TorchEmbedding # noqa - - return TorchEmbedding - elif name == 'ImageTorchTensor': - from docarray.typing.tensor.image import ImageTorchTensor # noqa - - return ImageTorchTensor - - elif name in tf_tensors: + elif 'TensorFlow' in name: import_library('tensorflow', raise_error=True) - if name == 'TensorFlowTensor': - from docarray.typing.tensor.tensorflow_tensor import ( # noqa - TensorFlowTensor, - ) + if name == 'TorchTensor': + import docarray.typing.tensor.torch_tensor as lib + elif name == 'TensorFlowTensor': + import docarray.typing.tensor.tensorflow_tensor as lib + elif name in ['TorchEmbedding', 'TensorFlowEmbedding']: + import docarray.typing.tensor.embedding as lib + elif name in ['ImageTorchTensor', 'ImageTensorFlowTensor']: + import docarray.typing.tensor.image as lib + elif name in ['AudioTorchTensor', 'AudioTensorFlowTensor']: + import docarray.typing.tensor.audio as lib + elif name in ['VideoTorchTensor', 'VideoTensorFlowTensor']: + import docarray.typing.tensor.video as lib + + tensor_cls = getattr(lib, name) - return TensorFlowTensor - elif name == 'TensorFlowEmbedding': - from docarray.typing.tensor.embedding import TensorFlowEmbedding # noqa - - return TensorFlowEmbedding - elif name == 'ImageTensorFlowTensor': - from docarray.typing.tensor.image import ImageTensorFlowTensor # noqa + if name not in __all__: + __all__.append(name) - return ImageTensorFlowTensor + return tensor_cls diff --git a/docarray/typing/tensor/audio/__init__.py b/docarray/typing/tensor/audio/__init__.py index 384fb90bb3d..f8c2a66b2f5 100644 --- a/docarray/typing/tensor/audio/__init__.py +++ b/docarray/typing/tensor/audio/__init__.py @@ -14,22 +14,17 @@ def __getattr__(name: str): - if name not in __all__: - __all__.append(name) - if name == 'AudioTorchTensor': import_library('torch', raise_error=True) - from docarray.typing.tensor.audio.audio_torch_tensor import ( # noqa - AudioTorchTensor, - ) - - return AudioTorchTensor - + import docarray.typing.tensor.audio.audio_torch_tensor as lib elif name == 'AudioTensorFlowTensor': import_library('tensorflow', raise_error=True) - from docarray.typing.tensor.audio.audio_tensorflow_tensor import ( # noqa - AudioTensorFlowTensor, - ) + import docarray.typing.tensor.audio.audio_tensorflow_tensor as lib + + tensor_cls = getattr(lib, name) + + if name not in __all__: + __all__.append(name) - return AudioTensorFlowTensor + return tensor_cls diff --git a/docarray/typing/tensor/embedding/__init__.py b/docarray/typing/tensor/embedding/__init__.py index fe1aca442a8..0e4617d69e3 100644 --- a/docarray/typing/tensor/embedding/__init__.py +++ b/docarray/typing/tensor/embedding/__init__.py @@ -12,20 +12,16 @@ def __getattr__(name: str): - if name not in __all__: - __all__.append(name) - if name == 'TorchEmbedding': import_library('torch', raise_error=True) - from docarray.typing.tensor.embedding.torch import TorchEmbedding # noqa - - return TorchEmbedding - + import docarray.typing.tensor.embedding.torch as lib elif name == 'TensorFlowEmbedding': import_library('tensorflow', raise_error=True) + import docarray.typing.tensor.embedding.tensorflow as lib + + tensor_cls = getattr(lib, name) - from docarray.typing.tensor.embedding.tensorflow import ( # noqa - TensorFlowEmbedding, - ) + if name not in __all__: + __all__.append(name) - return TensorFlowEmbedding + return tensor_cls diff --git a/docarray/typing/tensor/image/__init__.py b/docarray/typing/tensor/image/__init__.py index 1d8d7108b2a..0d446901ba6 100644 --- a/docarray/typing/tensor/image/__init__.py +++ b/docarray/typing/tensor/image/__init__.py @@ -14,22 +14,16 @@ def __getattr__(name: str): - if name not in __all__: - __all__.append(name) - if name == 'ImageTorchTensor': import_library('torch', raise_error=True) - from docarray.typing.tensor.image.image_torch_tensor import ( # noqa - ImageTorchTensor, - ) - - return ImageTorchTensor - + import docarray.typing.tensor.image.image_torch_tensor as lib elif name == 'ImageTensorFlowTensor': import_library('tensorflow', raise_error=True) + import docarray.typing.tensor.image.image_tensorflow_tensor as lib + + tensor_cls = getattr(lib, name) - from docarray.typing.tensor.image.image_tensorflow_tensor import ( # noqa - ImageTensorFlowTensor, - ) + if name not in __all__: + __all__.append(name) - return ImageTensorFlowTensor + return tensor_cls diff --git a/docarray/typing/tensor/video/__init__.py b/docarray/typing/tensor/video/__init__.py index 210dfcf8f6a..368a2650970 100644 --- a/docarray/typing/tensor/video/__init__.py +++ b/docarray/typing/tensor/video/__init__.py @@ -14,23 +14,16 @@ def __getattr__(name: str): - - if name not in __all__: - __all__.append(name) - if name == 'VideoTorchTensor': import_library('torch', raise_error=True) - from docarray.typing.tensor.video.video_torch_tensor import ( # noqa - VideoTorchTensor, - ) - - return VideoTorchTensor - + import docarray.typing.tensor.video.video_torch_tensor as lib elif name == 'VideoTensorFlowTensor': import_library('tensorflow', raise_error=True) + import docarray.typing.tensor.video.video_tensorflow_tensor as lib + + tensor_cls = getattr(lib, name) - from docarray.typing.tensor.video.video_tensorflow_tensor import ( # noqa - VideoTensorFlowTensor, - ) + if name not in __all__: + __all__.append(name) - return VideoTensorFlowTensor + return tensor_cls From 075ca2d37ce46f62e67cf81d7168093306264520 Mon Sep 17 00:00:00 2001 From: samsja <55492238+samsja@users.noreply.github.com> Date: Wed, 29 Mar 2023 13:17:06 +0200 Subject: [PATCH 49/66] fix: remove files (#1305) Signed-off-by: samsja Signed-off-by: anna-charlotte --- file_1.mp4 | Bin 8232 -> 0 bytes file_2.mp4 | Bin 8232 -> 0 bytes 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 file_1.mp4 delete mode 100644 file_2.mp4 diff --git a/file_1.mp4 b/file_1.mp4 deleted file mode 100644 index 15b58e871afd89ea39501de2fd6f2618a77bd9fe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8232 zcmeHMZ)hCV7QdTmW372rOSDz$W2EuVhh#Im*`#e?uxnadLBR@Ikq67&nY-CBGdq)+ zNxJciQHxgm@;*rArO#Ar_4%}-RQk$8eTh)vJrNO9w4&0|So~9JDQLPDUC+65?`CJ( z!t;ZEkR38}&%b-``Q0<;p4?eNh*}CKJS%V=LfQznaA)bbk#W3Sh7hu(Ax)o^{LW;nSy4Wi!L@7TDhC#UA` z9s-@o46qq;y$PF@LN%*teLYz%n+0Jx486hL-fi2qrN=Fkxi+n&U4OV2$4!?*#|9hM z3oW-245|h#QbW(EJ}c>cs>zDBYmDfbL2Xb|X@%Mo0n@c=-=J2_WcpNx>19?`gRvq| zYg8|ofUfW?_@U56H+Yeo%yX4_T()Y|G= z(+GhuMjc%P7pQr^TVZ-O)1AqvB^rbUFBq{rw1@^6^$KpO6tGb5$*N)5haLeqZ@caY zEdx_nkOj77fG<*zQMmxED?XphupAnqLslhZzD>azzZAw0M7!Fk|coRTdrk(|wn$$x$#X`}d0ZwB1j04=ptl@H41fA=#N@3XbKpP1? z=sLnCU~E0xuL-i?SQR~+Qv-ulm@yXWISt+PDXy*0f--dXjY0!A>U~3}MPGG_u)qkb zXn{Qf)LhpX;mLs2ZA7Eo}N=J526Qe3iN=n=_tGy7-ljF zFLRdd*iNpR`FZ|>o8H}jVyf%Fu@~m|OkZCZ8oMsFYv-X;N544B-aD}As?Sfa_DJgQ zM5I#Hel{Uy+2?uH$`50J&Tb5^NEEY~hgP17(xQYY;n zqB7OC?)ZugSI&@&KjExWi&(A&TjM@%`;oLA*o(G@r7dUMQM)`pRlfCweEDy;E$M%N zd;c0pJHeOdw@+=q^06moAKuDQE&7tnm0CjHhV~u#=|?tR-hK59A>(fWkLbTLw=93_ zr8*(6KFB3Xts&ty;N^2+{_#3Vy$yOq^v%`{D`!aClbls*5i7LQ|F{p97lxk&Z3h~U z@5Sj}o#Z@NM?BX9sC@v+UI2w5K~OIq0#N^30EL0+%qz8ooQDTMy|eL(IH(hQx7-;A z)sS#dm!zh@4&#oVd~!p~QNeRZ!Tj3&2a|YW{!*elDGsf zabfXl+UI4wc#%3!ekJ5GUXLVkL8(gY^9_l_1;s1oD3mLXOFAleY2K<|{GW|h{o?;` zwCb14FWlai!GB<_`X%=HR{e^~T~yEHKHsWeeBWr*FS&1EnZ(U@v;eR)1`bx$Yq;Iis0YE6=Oi0;MXr$o{wrH zr2R+uZaf0Yzl0-vPNm6Ej0W>2uf#m(o%aZI!kdsTW`+T@tC$_Y?;-qx5pdi^!smA1 zF)a!j)iIOUH4y)o+;xce>nWSI&nbA|?;euI_HO(gWHYUpHba~CYv6--35d>dhY#B9 zc9$9HR+;dX9XrE?`i?Q*RwH`x=RsJsfqpp%0~rvsc?y5*ffldkF(b(lfV~+=yzx5M zB9bpWSJvU%D(Qn!hf*f#tRE#ShN0bec<9mcdO{Qej}GPSrED@q7MH(0vA_GLGiR`S zCw%{pR`k-RflX6*tPDcF+CcKD!Ep#Q*a8TQ0$%76+{%r{f~^tBAR=71RJ4*u7W?qA zpl#`+^=Pe6v<~=s{E^;|C5Q)JcK*Jvg51IUz)R!~fxJRGwHI{vb4~OBxk7>Lhr7G! zz6I!<;CrE!Jkl-xw_Oj|`wSP>883_6!lg?<`FJvAx)o^{LW;nSy4Wi!L@7TDhC#UA` z9s-@o46qq;y$PF@LN%*teLYz%n+0Jx486hL-fi2qrN=Fkxi+n&U4OV2$4!?*#|9hM z3oW-245|h#QbW(EJ}c>cs>zDBYmDfbL2Xb|X@%Mo0n@c=-=J2_WcpNx>19?`gRvq| zYg8|ofUfW?_@U56H+Yeo%yX4_T()Y|G= z(+GhuMjc%P7pQr^TVZ-O)1AqvB^rbUFBq{rw1@^6^$KpO6tGb5$*N)5haLeqZ@caY zEdx_nkOj77fG<*zQMmxED?XphupAnqLslhZzD>azzZAw0M7!Fk|coRTdrk(|wn$$x$#X`}d0ZwB1j04=ptl@H41fA=#N@3XbKpP1? z=sLnCU~E0xuL-i?SQR~+Qv-ulm@yXWISt+PDXy*0f--dXjY0!A>U~3}MPGG_u)qkb zXn{Qf)LhpX;mLs2ZA7Eo}N=J526Qe3iN=n=_tGy7-ljF zFLRdd*iNpR`FZ|>o8H}jVyf%Fu@~m|OkZCZ8oMsFYv-X;N544B-aD}As?Sfa_DJgQ zM5I#Hel{Uy+2?uH$`50J&Tb5^NEEY~hgP17(xQYY;n zqB7OC?)ZugSI&@&KjExWi&(A&TjM@%`;oLA*o(G@r7dUMQM)`pRlfCweEDy;E$M%N zd;c0pJHeOdw@+=q^06moAKuDQE&7tnm0CjHhV~u#=|?tR-hK59A>(fWkLbTLw=93_ zr8*(6KFB3Xts&ty;N^2+{_#3Vy$yOq^v%`{D`!aClbls*5i7LQ|F{p97lxk&Z3h~U z@5Sj}o#Z@NM?BX9sC@v+UI2w5K~OIq0#N^30EL0+%qz8ooQDTMy|eL(IH(hQx7-;A z)sS#dm!zh@4&#oVd~!p~QNeRZ!Tj3&2a|YW{!*elDGsf zabfXl+UI4wc#%3!ekJ5GUXLVkL8(gY^9_l_1;s1oD3mLXOFAleY2K<|{GW|h{o?;` zwCb14FWlai!GB<_`X%=HR{e^~T~yEHKHsWeeBWr*FS&1EnZ(U@v;eR)1`bx$Yq;Iis0YE6=Oi0;MXr$o{wrH zr2R+uZaf0Yzl0-vPNm6Ej0W>2uf#m(o%aZI!kdsTW`+T@tC$_Y?;-qx5pdi^!smA1 zF)a!j)iIOUH4y)o+;xce>nWSI&nbA|?;euI_HO(gWHYUpHba~CYv6--35d>dhY#B9 zc9$9HR+;dX9XrE?`i?Q*RwH`x=RsJsfqpp%0~rvsc?y5*ffldkF(b(lfV~+=yzx5M zB9bpWSJvU%D(Qn!hf*f#tRE#ShN0bec<9mcdO{Qej}GPSrED@q7MH(0vA_GLGiR`S zCw%{pR`k-RflX6*tPDcF+CcKD!Ep#Q*a8TQ0$%76+{%r{f~^tBAR=71RJ4*u7W?qA zpl#`+^=Pe6v<~=s{E^;|C5Q)JcK*Jvg51IUz)R!~fxJRGwHI{vb4~OBxk7>Lhr7G! zz6I!<;CrE!Jkl-xw_Oj|`wSP>883_6!lg?<`FJv Date: Wed, 29 Mar 2023 19:36:51 +0800 Subject: [PATCH 50/66] fix: flatten schema of abstract index (#1294) * fix: flatten schema of abstract index Signed-off-by: AnneY * fix: _convert_dict_to_doc Signed-off-by: AnneY * fix: catch exception when flatten schema Signed-off-by: AnneY * refactor: remove useless assignemnt Signed-off-by: AnneY * fix: use Abstractensor as tensor doc_type Signed-off-by: AnneY * fix: add AbstractTensor to hnswlib Signed-off-by: AnneY * docs: AbstractTensor as doc_type Signed-off-by: AnneY * docs: complete description about AbstracTensor Co-authored-by: Johannes Messner <44071807+JohannesMessner@users.noreply.github.com> Signed-off-by: Anne Yang --------- Signed-off-by: AnneY Signed-off-by: Anne Yang Co-authored-by: Johannes Messner <44071807+JohannesMessner@users.noreply.github.com> --- docarray/index/abstract.py | 95 +++++++++------ docarray/index/backends/hnswlib.py | 3 +- docarray/utils/_internal/_typing.py | 16 +-- docs/how_to/add_doc_index.md | 7 +- .../index/base_classes/test_base_doc_store.py | 114 +++++++++++++++--- 5 files changed, 162 insertions(+), 73 deletions(-) diff --git a/docarray/index/abstract.py b/docarray/index/abstract.py index 1f4deceaa8b..89d8573bd23 100644 --- a/docarray/index/abstract.py +++ b/docarray/index/abstract.py @@ -26,7 +26,8 @@ from docarray import BaseDoc, DocArray from docarray.array.abstract_array import AnyDocArray from docarray.typing import AnyTensor -from docarray.utils._internal._typing import unwrap_optional_type +from docarray.utils._internal._typing import is_tensor_union +from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.utils._internal.misc import import_library from docarray.utils.find import FindResult, _FindResult @@ -677,22 +678,37 @@ def _flatten_schema( if is_union_type(t_): union_args = get_args(t_) - if len(union_args) == 2 and type(None) in union_args: + + if is_tensor_union(t_): + names_types_fields.append( + (name_prefix + field_name, AbstractTensor, field_) + ) + + elif len(union_args) == 2 and type(None) in union_args: # simple "Optional" type, treat as special case: # treat as if it was a single non-optional type for t_arg in union_args: - if t_arg is type(None): - pass - elif issubclass(t_arg, BaseDoc): - names_types_fields.extend( - cls._flatten_schema(t_arg, name_prefix=inner_prefix) - ) + if t_arg is not type(None): + if issubclass(t_arg, BaseDoc): + names_types_fields.extend( + cls._flatten_schema(t_arg, name_prefix=inner_prefix) + ) + else: + names_types_fields.append( + (name_prefix + field_name, t_arg, field_) + ) else: - names_types_fields.append((field_name, t_, field_)) + raise ValueError( + f'Union type {t_} is not supported. Only Union of subclasses of AbstractTensor or Union[type, None] are supported.' + ) elif issubclass(t_, BaseDoc): names_types_fields.extend( cls._flatten_schema(t_, name_prefix=inner_prefix) ) + elif issubclass(t_, AbstractTensor): + names_types_fields.append( + (name_prefix + field_name, AbstractTensor, field_) + ) else: names_types_fields.append((name_prefix + field_name, t_, field_)) return names_types_fields @@ -706,16 +722,8 @@ def _create_column_infos(self, schema: Type[BaseDoc]) -> Dict[str, _ColumnInfo]: """ column_infos: Dict[str, _ColumnInfo] = dict() for field_name, type_, field_ in self._flatten_schema(schema): - if is_optional_type(type_): - column_infos[field_name] = self._create_single_column( - field_, unwrap_optional_type(type_) - ) - elif is_union_type(type_): - raise ValueError( - 'Union types are not supported in the schema of a DocumentIndex.' - f' Instead of using type {type_} use a single specific type.' - ) - elif issubclass(type_, AnyDocArray): + # Union types are handle in _flatten_schema + if issubclass(type_, AnyDocArray): raise ValueError( 'Indexing field of DocArray type (=subindex)' 'is not yet supported.' @@ -726,7 +734,6 @@ def _create_column_infos(self, schema: Type[BaseDoc]) -> Dict[str, _ColumnInfo]: def _create_single_column(self, field: 'ModelField', type_: Type) -> _ColumnInfo: custom_config = field.field_info.extra - if 'col_type' in custom_config.keys(): db_type = custom_config['col_type'] custom_config.pop('col_type') @@ -741,13 +748,13 @@ def _create_single_column(self, field: 'ModelField', type_: Type) -> _ColumnInfo config.update(custom_config) # parse n_dim from parametrized tensor type if ( - hasattr(type_, '__docarray_target_shape__') - and type_.__docarray_target_shape__ + hasattr(field.type_, '__docarray_target_shape__') + and field.type_.__docarray_target_shape__ ): - if len(type_.__docarray_target_shape__) == 1: - n_dim = type_.__docarray_target_shape__[0] + if len(field.type_.__docarray_target_shape__) == 1: + n_dim = field.type_.__docarray_target_shape__[0] else: - n_dim = type_.__docarray_target_shape__ + n_dim = field.type_.__docarray_target_shape__ else: n_dim = None return _ColumnInfo( @@ -777,19 +784,23 @@ def _validate_docs( ) reference_names = [name for (name, _, _) in reference_schema_flat] reference_types = [t_ for (_, t_, _) in reference_schema_flat] + try: + input_schema_flat = self._flatten_schema(docs.document_type) + except ValueError: + pass + else: + input_names = [name for (name, _, _) in input_schema_flat] + input_types = [t_ for (_, t_, _) in input_schema_flat] + # this could be relaxed in the future, + # see schema translation ideas in the design doc + names_compatible = reference_names == input_names + types_compatible = all( + (issubclass(t2, t1)) + for (t1, t2) in zip(reference_types, input_types) + ) + if names_compatible and types_compatible: + return docs - input_schema_flat = self._flatten_schema(docs.document_type) - input_names = [name for (name, _, _) in input_schema_flat] - input_types = [t_ for (_, t_, _) in input_schema_flat] - # this could be relaxed in the future, - # see schema translation ideas in the design doc - names_compatible = reference_names == input_names - types_compatible = all( - (not is_union_type(t2) and issubclass(t1, t2)) - for (t1, t2) in zip(reference_types, input_types) - ) - if names_compatible and types_compatible: - return docs out_docs = [] for i in range(len(docs)): # validate the data @@ -837,10 +848,14 @@ def _convert_dict_to_doc( :param schema: The schema of the Document object :return: A Document object """ - for field_name, _ in schema.__fields__.items(): - t_ = unwrap_optional_type(schema._get_field_type(field_name)) - if issubclass(t_, BaseDoc): + t_ = schema._get_field_type(field_name) + if is_optional_type(t_): + for t_arg in get_args(t_): + if t_arg is not type(None): + t_ = t_arg + + if not is_union_type(t_) and issubclass(t_, BaseDoc): inner_dict = {} fields = [ diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index 220c55cf138..c234d271c3c 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -29,6 +29,7 @@ _raise_not_supported, ) from docarray.proto import DocumentProto +from docarray.typing.tensor.abstract_tensor import AbstractTensor from docarray.utils._internal.misc import import_library, is_np_int from docarray.utils.filter import filter_docs from docarray.utils.find import _FindResult @@ -46,7 +47,7 @@ if tf is not None: from docarray.typing import TensorFlowTensor -HNSWLIB_PY_VEC_TYPES = [list, tuple, np.ndarray] +HNSWLIB_PY_VEC_TYPES = [list, tuple, np.ndarray, AbstractTensor] if torch is not None: HNSWLIB_PY_VEC_TYPES.append(torch.Tensor) diff --git a/docarray/utils/_internal/_typing.py b/docarray/utils/_internal/_typing.py index 9bbc0162432..62680cf964e 100644 --- a/docarray/utils/_internal/_typing.py +++ b/docarray/utils/_internal/_typing.py @@ -1,6 +1,6 @@ from typing import Any, Optional -from typing_inspect import get_args, is_optional_type, is_union_type +from typing_inspect import get_args, is_union_type from docarray.typing.tensor.abstract_tensor import AbstractTensor @@ -32,17 +32,3 @@ def change_cls_name(cls: type, new_name: str, scope: Optional[dict] = None) -> N scope[new_name] = cls cls.__qualname__ = cls.__qualname__[: -len(cls.__name__)] + new_name cls.__name__ = new_name - - -def unwrap_optional_type(type_: Any) -> Any: - """Return the type of an Optional type, e.g. `unwrap_optional(Optional[str]) == str`; - `unwrap_optional(Union[None, int, None]) == int`. - - :param type_: the type to unwrap - :return: the "core" type of an Optional type - """ - if not is_optional_type(type_): - return type_ - for arg in get_args(type_): - if arg is not type(None): - return arg diff --git a/docs/how_to/add_doc_index.md b/docs/how_to/add_doc_index.md index 054a2b94cef..3d4cfb9b8bc 100644 --- a/docs/how_to/add_doc_index.md +++ b/docs/how_to/add_doc_index.md @@ -180,7 +180,7 @@ class _ColumnInfo: config: Dict[str, Any] ``` -- `docarray_type` is the type of the column in DocArray, e.g. `NdArray` or `str` +- `docarray_type` is the type of the column in DocArray, e.g. `AbstractTensor` or `str` - `db_type` is the type of the column in the Document Index, e.g. `np.ndarray` or `str`. You can customize the mapping from `docarray_type` to `db_type`, as we will see later. - `config` is a dictionary of configurations for the column. For example, for the `other_tensor` column above, this would contain the `space` and `dim` configurations. - `n_dim` is the dimensionality of the column, e.g. `100` for a 100-dimensional vector. See further guidance on this below. @@ -194,6 +194,9 @@ By default, it holds that `_ColumnInfo.docarray_type == self.python_type_to_db_t However, you should not rely on this, because a user can manually specify a different db_type. Therefore, your implementation should rely on `_ColumnInfo.db_type` and not directly call `python_type_to_db_type()`. +**Caution** +If a subclass of `AbstractTensor` appears in the Document Index's schema (i.e. `TorchTensor`, `NdArray`, or `TensorFlowTensor`), then `_ColumnInfo.docarray_type` will simply show `AbstractTensor` instead of the specific subclass. This is because the abstract class normalizes all input data of type `AbstractTensor` to `np.ndarray` anyways, which should make your life easier. Just be sure to properly handle `AbstractTensor` as a possible value or `_ColumnInfo.docarray_type`, and you won't have to worry about the differences between torch, tf, and np. + ### Properly handle `n_dim` `_ColumnInfo.n_dim` is automatically obtained from type parametrizations of the form `NdArray[100]`; @@ -337,7 +340,7 @@ The details of each method should become clear from the docstrings and type hint This method is slightly special, because 1) it is not exposed to the user, and 2) you absolutely have to implement it. -It is intended to do the following: It takes a type of a field in the store's schema (e.g. `NdArray` for `tensor`), and returns the corresponding type in the database (e.g. `np.ndarray`). +It is intended to do the following: It takes a type of a field in the store's schema (e.g. `AbstractTensor` for `tensor`), and returns the corresponding type in the database (e.g. `np.ndarray`). The `BaseDocIndex` class uses this information to create and populate the `_ColumnInfo`s in `self._column_infos`. If the user wants to change the default behaviour, one can set the db type by using the `col_type` field: diff --git a/tests/index/base_classes/test_base_doc_store.py b/tests/index/base_classes/test_base_doc_store.py index cd1d8339ab2..b5774020524 100644 --- a/tests/index/base_classes/test_base_doc_store.py +++ b/tests/index/base_classes/test_base_doc_store.py @@ -6,8 +6,11 @@ from pydantic import Field from docarray import BaseDoc, DocArray +from docarray.documents import ImageDoc from docarray.index.abstract import BaseDocIndex, _raise_not_composable -from docarray.typing import ID, NdArray +from docarray.typing import ID, ImageBytes, ImageUrl, NdArray +from docarray.typing.tensor.abstract_tensor import AbstractTensor +from docarray.utils._internal.misc import torch_imported pytestmark = pytest.mark.index @@ -45,6 +48,7 @@ class RuntimeConfig(BaseDocIndex.RuntimeConfig): str: {'hi': 'there'}, np.ndarray: {'you': 'good?'}, 'varchar': {'good': 'bye'}, + AbstractTensor: {'dim': 1000}, } ) @@ -103,7 +107,7 @@ def test_create_columns(): assert store._column_infos['id'].n_dim is None assert store._column_infos['id'].config == {'hi': 'there'} - assert issubclass(store._column_infos['tens'].docarray_type, NdArray) + assert issubclass(store._column_infos['tens'].docarray_type, AbstractTensor) assert store._column_infos['tens'].db_type == str assert store._column_infos['tens'].n_dim == 10 assert store._column_infos['tens'].config == {'dim': 1000, 'hi': 'there'} @@ -117,12 +121,12 @@ def test_create_columns(): assert store._column_infos['id'].n_dim is None assert store._column_infos['id'].config == {'hi': 'there'} - assert issubclass(store._column_infos['tens_one'].docarray_type, NdArray) + assert issubclass(store._column_infos['tens_one'].docarray_type, AbstractTensor) assert store._column_infos['tens_one'].db_type == str assert store._column_infos['tens_one'].n_dim is None assert store._column_infos['tens_one'].config == {'dim': 10, 'hi': 'there'} - assert issubclass(store._column_infos['tens_two'].docarray_type, NdArray) + assert issubclass(store._column_infos['tens_two'].docarray_type, AbstractTensor) assert store._column_infos['tens_two'].db_type == str assert store._column_infos['tens_two'].n_dim is None assert store._column_infos['tens_two'].config == {'dim': 50, 'hi': 'there'} @@ -136,7 +140,7 @@ def test_create_columns(): assert store._column_infos['id'].n_dim is None assert store._column_infos['id'].config == {'hi': 'there'} - assert issubclass(store._column_infos['d__tens'].docarray_type, NdArray) + assert issubclass(store._column_infos['d__tens'].docarray_type, AbstractTensor) assert store._column_infos['d__tens'].db_type == str assert store._column_infos['d__tens'].n_dim == 10 assert store._column_infos['d__tens'].config == {'dim': 1000, 'hi': 'there'} @@ -147,15 +151,15 @@ def test_flatten_schema(): fields = SimpleDoc.__fields__ assert set(store._flatten_schema(SimpleDoc)) == { ('id', ID, fields['id']), - ('tens', NdArray[10], fields['tens']), + ('tens', AbstractTensor, fields['tens']), } store = DummyDocIndex[FlatDoc]() fields = FlatDoc.__fields__ assert set(store._flatten_schema(FlatDoc)) == { ('id', ID, fields['id']), - ('tens_one', NdArray, fields['tens_one']), - ('tens_two', NdArray, fields['tens_two']), + ('tens_one', AbstractTensor, fields['tens_one']), + ('tens_two', AbstractTensor, fields['tens_two']), } store = DummyDocIndex[NestedDoc]() @@ -164,7 +168,7 @@ def test_flatten_schema(): assert set(store._flatten_schema(NestedDoc)) == { ('id', ID, fields['id']), ('d__id', ID, fields_nested['id']), - ('d__tens', NdArray[10], fields_nested['tens']), + ('d__tens', AbstractTensor, fields_nested['tens']), } store = DummyDocIndex[DeepNestedDoc]() @@ -175,7 +179,44 @@ def test_flatten_schema(): ('id', ID, fields['id']), ('d__id', ID, fields_nested['id']), ('d__d__id', ID, fields_nested_nested['id']), - ('d__d__tens', NdArray[10], fields_nested_nested['tens']), + ('d__d__tens', AbstractTensor, fields_nested_nested['tens']), + } + + +def test_flatten_schema_union(): + class MyDoc(BaseDoc): + image: ImageDoc + + store = DummyDocIndex[MyDoc]() + fields = MyDoc.__fields__ + fields_image = ImageDoc.__fields__ + + if torch_imported: + from docarray.typing.tensor.image.image_torch_tensor import ImageTorchTensor + + assert set(store._flatten_schema(MyDoc)) == { + ('id', ID, fields['id']), + ('image__id', ID, fields_image['id']), + ('image__url', ImageUrl, fields_image['url']), + ('image__tensor', AbstractTensor, fields_image['tensor']), + ('image__embedding', AbstractTensor, fields_image['embedding']), + ('image__bytes_', ImageBytes, fields_image['bytes_']), + } + + class MyDoc2(BaseDoc): + tensor: Union[NdArray, str] + + with pytest.raises(ValueError): + _ = DummyDocIndex[MyDoc2]() + + class MyDoc3(BaseDoc): + tensor: Union[NdArray, ImageTorchTensor] + + store = DummyDocIndex[MyDoc3]() + fields = MyDoc3.__fields__ + assert set(store._flatten_schema(MyDoc3)) == { + ('id', ID, fields['id']), + ('tensor', AbstractTensor, fields['tensor']), } @@ -303,9 +344,12 @@ def test_docs_validation_unions(): class OptionalDoc(BaseDoc): tens: Optional[NdArray[10]] = Field(dim=1000) - class UnionDoc(BaseDoc): + class MixedUnionDoc(BaseDoc): tens: Union[NdArray[10], str] = Field(dim=1000) + class TensorUnionDoc(BaseDoc): + tens: Union[NdArray[10], AbstractTensor] = Field(dim=1000) + # OPTIONAL store = DummyDocIndex[SimpleDoc]() in_list = [OptionalDoc(tens=np.random.random((10,)))] @@ -316,15 +360,28 @@ class UnionDoc(BaseDoc): with pytest.raises(ValueError): store._validate_docs([OptionalDoc(tens=None)]) - # OTHER UNION + # MIXED UNION store = DummyDocIndex[SimpleDoc]() - in_list = [UnionDoc(tens=np.random.random((10,)))] + in_list = [MixedUnionDoc(tens=np.random.random((10,)))] assert isinstance(store._validate_docs(in_list), DocArray[BaseDoc]) - in_da = DocArray[UnionDoc](in_list) + in_da = DocArray[MixedUnionDoc](in_list) assert isinstance(store._validate_docs(in_da), DocArray[BaseDoc]) with pytest.raises(ValueError): - store._validate_docs([UnionDoc(tens='hello')]) + store._validate_docs([MixedUnionDoc(tens='hello')]) + + # TENSOR UNION + store = DummyDocIndex[TensorUnionDoc]() + in_list = [SimpleDoc(tens=np.random.random((10,)))] + assert isinstance(store._validate_docs(in_list), DocArray[BaseDoc]) + in_da = DocArray[SimpleDoc](in_list) + assert store._validate_docs(in_da) == in_da + + store = DummyDocIndex[SimpleDoc]() + in_list = [TensorUnionDoc(tens=np.random.random((10,)))] + assert isinstance(store._validate_docs(in_list), DocArray[BaseDoc]) + in_da = DocArray[TensorUnionDoc](in_list) + assert store._validate_docs(in_da) == in_da def test_get_value(): @@ -474,3 +531,30 @@ def test_convert_dict_to_doc(): assert doc.d.id == doc_dict_copy['d__id'] assert doc.d.d.id == doc_dict_copy['d__d__id'] assert np.all(doc.d.d.tens == doc_dict_copy['d__d__tens']) + + class MyDoc(BaseDoc): + image: ImageDoc + + store = DummyDocIndex[MyDoc]() + doc_dict = { + 'id': 'root', + 'image__id': 'nested', + 'image__tensor': np.random.random((128,)), + } + doc = store._convert_dict_to_doc(doc_dict, store._schema) + + if torch_imported: + from docarray.typing.tensor.image.image_torch_tensor import ImageTorchTensor + + class MyDoc2(BaseDoc): + tens: Union[NdArray, ImageTorchTensor] + + store = DummyDocIndex[MyDoc2]() + doc_dict = { + 'id': 'root', + 'tens': np.random.random((128,)), + } + doc_dict_copy = doc_dict.copy() + doc = store._convert_dict_to_doc(doc_dict, store._schema) + assert doc.id == doc_dict_copy['id'] + assert np.all(doc.tens == doc_dict_copy['tens']) From fde056d5b090ac68a9a3c180a6459e871b6ea0df Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Wed, 29 Mar 2023 14:38:30 +0200 Subject: [PATCH 51/66] fix: add type hint for lib Signed-off-by: anna-charlotte --- docarray/store/__init__.py | 2 ++ docarray/typing/tensor/__init__.py | 3 +++ docarray/typing/tensor/audio/__init__.py | 3 ++- docarray/typing/tensor/embedding/__init__.py | 3 +++ docarray/typing/tensor/image/__init__.py | 3 +++ docarray/typing/tensor/video/__init__.py | 3 +++ 6 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docarray/store/__init__.py b/docarray/store/__init__.py index 2c005ea1bfd..04a6e61a6f7 100644 --- a/docarray/store/__init__.py +++ b/docarray/store/__init__.py @@ -1,3 +1,4 @@ +import types from typing import TYPE_CHECKING from docarray.store.file import FileDocStore @@ -11,6 +12,7 @@ def __getattr__(name: str): + lib: types.ModuleType if name == 'JACDocStore': import_library('hubble', raise_error=True) import docarray.store.jac as lib diff --git a/docarray/typing/tensor/__init__.py b/docarray/typing/tensor/__init__.py index 6377712bd14..02db4b23495 100644 --- a/docarray/typing/tensor/__init__.py +++ b/docarray/typing/tensor/__init__.py @@ -1,3 +1,5 @@ +import types + from typing_extensions import TYPE_CHECKING from docarray.typing.tensor.audio import AudioNdArray @@ -38,6 +40,7 @@ def __getattr__(name: str): elif 'TensorFlow' in name: import_library('tensorflow', raise_error=True) + lib: types.ModuleType if name == 'TorchTensor': import docarray.typing.tensor.torch_tensor as lib elif name == 'TensorFlowTensor': diff --git a/docarray/typing/tensor/audio/__init__.py b/docarray/typing/tensor/audio/__init__.py index f8c2a66b2f5..fd048c03119 100644 --- a/docarray/typing/tensor/audio/__init__.py +++ b/docarray/typing/tensor/audio/__init__.py @@ -1,3 +1,4 @@ +import types from typing import TYPE_CHECKING from docarray.typing.tensor.audio.audio_ndarray import AudioNdArray @@ -14,12 +15,12 @@ def __getattr__(name: str): + lib: types.ModuleType if name == 'AudioTorchTensor': import_library('torch', raise_error=True) import docarray.typing.tensor.audio.audio_torch_tensor as lib elif name == 'AudioTensorFlowTensor': import_library('tensorflow', raise_error=True) - import docarray.typing.tensor.audio.audio_tensorflow_tensor as lib tensor_cls = getattr(lib, name) diff --git a/docarray/typing/tensor/embedding/__init__.py b/docarray/typing/tensor/embedding/__init__.py index 0e4617d69e3..b5279e9cc20 100644 --- a/docarray/typing/tensor/embedding/__init__.py +++ b/docarray/typing/tensor/embedding/__init__.py @@ -1,3 +1,5 @@ +import types + from typing_extensions import TYPE_CHECKING from docarray.typing.tensor.embedding.embedding import AnyEmbedding @@ -12,6 +14,7 @@ def __getattr__(name: str): + lib: types.ModuleType if name == 'TorchEmbedding': import_library('torch', raise_error=True) import docarray.typing.tensor.embedding.torch as lib diff --git a/docarray/typing/tensor/image/__init__.py b/docarray/typing/tensor/image/__init__.py index 0d446901ba6..c0b490b0617 100644 --- a/docarray/typing/tensor/image/__init__.py +++ b/docarray/typing/tensor/image/__init__.py @@ -1,3 +1,5 @@ +import types + from typing_extensions import TYPE_CHECKING from docarray.typing.tensor.image.image_ndarray import ImageNdArray @@ -14,6 +16,7 @@ def __getattr__(name: str): + lib: types.ModuleType if name == 'ImageTorchTensor': import_library('torch', raise_error=True) import docarray.typing.tensor.image.image_torch_tensor as lib diff --git a/docarray/typing/tensor/video/__init__.py b/docarray/typing/tensor/video/__init__.py index 368a2650970..26b74758819 100644 --- a/docarray/typing/tensor/video/__init__.py +++ b/docarray/typing/tensor/video/__init__.py @@ -1,3 +1,5 @@ +import types + from typing_extensions import TYPE_CHECKING from docarray.typing.tensor.video.video_ndarray import VideoNdArray @@ -14,6 +16,7 @@ def __getattr__(name: str): + lib: types.ModuleType if name == 'VideoTorchTensor': import_library('torch', raise_error=True) import docarray.typing.tensor.video.video_torch_tensor as lib From ec5c4f4f8fa35bd40ece42187acde9fa8c278e1b Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Wed, 29 Mar 2023 15:44:52 +0200 Subject: [PATCH 52/66] fix: add import error to inits getattrs Signed-off-by: anna-charlotte --- docarray/base_doc/__init__.py | 9 +++++- docarray/index/__init__.py | 9 +++++- docarray/store/__init__.py | 9 +++++- docarray/typing/__init__.py | 29 ++++++++++++++++++-- docarray/typing/tensor/__init__.py | 9 +++++- docarray/typing/tensor/audio/__init__.py | 9 +++++- docarray/typing/tensor/embedding/__init__.py | 9 +++++- docarray/typing/tensor/image/__init__.py | 9 +++++- docarray/typing/tensor/video/__init__.py | 9 +++++- docarray/utils/_internal/misc.py | 8 ++++++ 10 files changed, 98 insertions(+), 11 deletions(-) diff --git a/docarray/base_doc/__init__.py b/docarray/base_doc/__init__.py index e835671439e..941e76c542b 100644 --- a/docarray/base_doc/__init__.py +++ b/docarray/base_doc/__init__.py @@ -1,7 +1,10 @@ from docarray.base_doc.any_doc import AnyDoc from docarray.base_doc.base_node import BaseNode from docarray.base_doc.doc import BaseDoc -from docarray.utils._internal.misc import import_library +from docarray.utils._internal.misc import ( + _get_path_from_docarray_root_level, + import_library, +) __all__ = ['AnyDoc', 'BaseDoc', 'BaseNode'] @@ -15,3 +18,7 @@ def __getattr__(name: str): __all__.append(name) return DocResponse + else: + raise ImportError( + f'cannot import name \'{name}\' from \'{_get_path_from_docarray_root_level(__file__)}\'' + ) diff --git a/docarray/index/__init__.py b/docarray/index/__init__.py index 2ddef0931af..6b2ed9e9f8c 100644 --- a/docarray/index/__init__.py +++ b/docarray/index/__init__.py @@ -1,6 +1,9 @@ from typing import TYPE_CHECKING -from docarray.utils._internal.misc import import_library +from docarray.utils._internal.misc import ( + _get_path_from_docarray_root_level, + import_library, +) if TYPE_CHECKING: from docarray.index.backends.hnswlib import HnswDocumentIndex # noqa: F401 @@ -17,3 +20,7 @@ def __getattr__(name: str): __all__.append(name) return HnswDocumentIndex + else: + raise ImportError( + f'cannot import name \'{name}\' from \'{_get_path_from_docarray_root_level(__file__)}\'' + ) diff --git a/docarray/store/__init__.py b/docarray/store/__init__.py index 04a6e61a6f7..e21892bcdd6 100644 --- a/docarray/store/__init__.py +++ b/docarray/store/__init__.py @@ -2,7 +2,10 @@ from typing import TYPE_CHECKING from docarray.store.file import FileDocStore -from docarray.utils._internal.misc import import_library +from docarray.utils._internal.misc import ( + _get_path_from_docarray_root_level, + import_library, +) if TYPE_CHECKING: from docarray.store.jac import JACDocStore # noqa: F401 @@ -21,6 +24,10 @@ def __getattr__(name: str): import_library('botocore', raise_error=True) import_library('boto3', raise_error=True) import docarray.store.s3 as lib + else: + raise ImportError( + f'cannot import name \'{name}\' from \'{_get_path_from_docarray_root_level(__file__)}\'' + ) tensor_cls = getattr(lib, name) diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py index b00dcdf45d1..f729b985456 100644 --- a/docarray/typing/__init__.py +++ b/docarray/typing/__init__.py @@ -17,7 +17,10 @@ TextUrl, VideoUrl, ) -from docarray.utils._internal.misc import import_library +from docarray.utils._internal.misc import ( + _get_path_from_docarray_root_level, + import_library, +) if TYPE_CHECKING: from docarray.typing.tensor import TensorFlowTensor # noqa: F401 @@ -54,11 +57,31 @@ ] +_torch_tensors = [ + 'TorchTensor', + 'TorchEmbedding', + 'ImageTorchTensor', + 'AudioTorchTensor', + 'VideoTorchTensor', +] +_tf_tensors = [ + 'TensorFlowTensor', + 'TensorFlowEmbedding', + 'ImageTensorFlowTensor', + 'AudioTensorFlowTensor', + 'VideoTensorFlowTensor', +] + + def __getattr__(name: str): - if 'Torch' in name: + if name in _torch_tensors: import_library('torch', raise_error=True) - elif 'TensorFlow' in name: + elif name in _tf_tensors: import_library('tensorflow', raise_error=True) + else: + raise ImportError( + f'cannot import name \'{name}\' from \'{_get_path_from_docarray_root_level(__file__)}\'' + ) import docarray.typing.tensor diff --git a/docarray/typing/tensor/__init__.py b/docarray/typing/tensor/__init__.py index 02db4b23495..4c4077f3cdb 100644 --- a/docarray/typing/tensor/__init__.py +++ b/docarray/typing/tensor/__init__.py @@ -8,7 +8,10 @@ from docarray.typing.tensor.ndarray import NdArray from docarray.typing.tensor.tensor import AnyTensor from docarray.typing.tensor.video import VideoNdArray -from docarray.utils._internal.misc import import_library +from docarray.utils._internal.misc import ( + _get_path_from_docarray_root_level, + import_library, +) if TYPE_CHECKING: from docarray.typing.tensor.audio import AudioTensorFlowTensor # noqa: F401 @@ -53,6 +56,10 @@ def __getattr__(name: str): import docarray.typing.tensor.audio as lib elif name in ['VideoTorchTensor', 'VideoTensorFlowTensor']: import docarray.typing.tensor.video as lib + else: + raise ImportError( + f'cannot import name \'{name}\' from \'{_get_path_from_docarray_root_level(__file__)}\'' + ) tensor_cls = getattr(lib, name) diff --git a/docarray/typing/tensor/audio/__init__.py b/docarray/typing/tensor/audio/__init__.py index fd048c03119..a505ab05720 100644 --- a/docarray/typing/tensor/audio/__init__.py +++ b/docarray/typing/tensor/audio/__init__.py @@ -3,7 +3,10 @@ from docarray.typing.tensor.audio.audio_ndarray import AudioNdArray from docarray.typing.tensor.audio.audio_tensor import AudioTensor -from docarray.utils._internal.misc import import_library +from docarray.utils._internal.misc import ( + _get_path_from_docarray_root_level, + import_library, +) if TYPE_CHECKING: from docarray.typing.tensor.audio.audio_tensorflow_tensor import ( # noqa @@ -22,6 +25,10 @@ def __getattr__(name: str): elif name == 'AudioTensorFlowTensor': import_library('tensorflow', raise_error=True) import docarray.typing.tensor.audio.audio_tensorflow_tensor as lib + else: + raise ImportError( + f'cannot import name \'{name}\' from \'{_get_path_from_docarray_root_level(__file__)}\'' + ) tensor_cls = getattr(lib, name) diff --git a/docarray/typing/tensor/embedding/__init__.py b/docarray/typing/tensor/embedding/__init__.py index b5279e9cc20..c32048b21c6 100644 --- a/docarray/typing/tensor/embedding/__init__.py +++ b/docarray/typing/tensor/embedding/__init__.py @@ -4,7 +4,10 @@ from docarray.typing.tensor.embedding.embedding import AnyEmbedding from docarray.typing.tensor.embedding.ndarray import NdArrayEmbedding -from docarray.utils._internal.misc import import_library +from docarray.utils._internal.misc import ( + _get_path_from_docarray_root_level, + import_library, +) if TYPE_CHECKING: from docarray.typing.tensor.embedding.tensorflow import TensorFlowEmbedding # noqa @@ -21,6 +24,10 @@ def __getattr__(name: str): elif name == 'TensorFlowEmbedding': import_library('tensorflow', raise_error=True) import docarray.typing.tensor.embedding.tensorflow as lib + else: + raise ImportError( + f'cannot import name \'{name}\' from \'{_get_path_from_docarray_root_level(__file__)}\'' + ) tensor_cls = getattr(lib, name) diff --git a/docarray/typing/tensor/image/__init__.py b/docarray/typing/tensor/image/__init__.py index c0b490b0617..7af4b852206 100644 --- a/docarray/typing/tensor/image/__init__.py +++ b/docarray/typing/tensor/image/__init__.py @@ -4,7 +4,10 @@ from docarray.typing.tensor.image.image_ndarray import ImageNdArray from docarray.typing.tensor.image.image_tensor import ImageTensor -from docarray.utils._internal.misc import import_library +from docarray.utils._internal.misc import ( + _get_path_from_docarray_root_level, + import_library, +) if TYPE_CHECKING: from docarray.typing.tensor.image.image_tensorflow_tensor import ( # noqa @@ -23,6 +26,10 @@ def __getattr__(name: str): elif name == 'ImageTensorFlowTensor': import_library('tensorflow', raise_error=True) import docarray.typing.tensor.image.image_tensorflow_tensor as lib + else: + raise ImportError( + f'cannot import name \'{name}\' from \'{_get_path_from_docarray_root_level(__file__)}\'' + ) tensor_cls = getattr(lib, name) diff --git a/docarray/typing/tensor/video/__init__.py b/docarray/typing/tensor/video/__init__.py index 26b74758819..a575e7b6201 100644 --- a/docarray/typing/tensor/video/__init__.py +++ b/docarray/typing/tensor/video/__init__.py @@ -4,7 +4,10 @@ from docarray.typing.tensor.video.video_ndarray import VideoNdArray from docarray.typing.tensor.video.video_tensor import VideoTensor -from docarray.utils._internal.misc import import_library +from docarray.utils._internal.misc import ( + _get_path_from_docarray_root_level, + import_library, +) if TYPE_CHECKING: from docarray.typing.tensor.video.video_tensorflow_tensor import ( # noqa @@ -23,6 +26,10 @@ def __getattr__(name: str): elif name == 'VideoTensorFlowTensor': import_library('tensorflow', raise_error=True) import docarray.typing.tensor.video.video_tensorflow_tensor as lib + else: + raise ImportError( + f'cannot import name \'{name}\' from \'{_get_path_from_docarray_root_level(__file__)}\'' + ) tensor_cls = getattr(lib, name) diff --git a/docarray/utils/_internal/misc.py b/docarray/utils/_internal/misc.py index e3262cd067e..362507d264d 100644 --- a/docarray/utils/_internal/misc.py +++ b/docarray/utils/_internal/misc.py @@ -1,4 +1,6 @@ import importlib +import os +import re import types from typing import Any, Optional @@ -57,6 +59,12 @@ def import_library( return lib +def _get_path_from_docarray_root_level(file_path: str) -> str: + path = os.path.dirname(file_path) + rel_path = re.sub('(?s:.*)docarray', 'docarray', path).replace('/', '.') + return rel_path + + def is_torch_available(): return torch_imported From 550981de89c9be049b6215c13b91d002e87ca08f Mon Sep 17 00:00:00 2001 From: samsja <55492238+samsja@users.noreply.github.com> Date: Wed, 29 Mar 2023 14:32:35 +0200 Subject: [PATCH 53/66] docs: add utils section (#1307) * feat: add utils for map to docs and fix docstring Signed-off-by: samsja * feat: add utils for map to docs and fix docstring Signed-off-by: samsja * feat: add utils for find and fix docstring Signed-off-by: samsja * fix: fix video ndaray docstrng Signed-off-by: samsja * fix: fix video find docstrng Signed-off-by: samsja * fix: fix map docstring Signed-off-by: samsja * fix: fix fileter docstring Signed-off-by: samsja * fix: fix add reduce Signed-off-by: samsja --------- Signed-off-by: samsja Signed-off-by: anna-charlotte --- docarray/typing/tensor/video/video_ndarray.py | 6 +- docarray/utils/filter.py | 79 +++++---- docarray/utils/find.py | 165 +++++++++--------- docarray/utils/map.py | 114 ++++++------ docarray/utils/reduce.py | 2 + docs/api_references/utils/filter.md | 7 + docs/api_references/utils/find.md | 8 + docs/api_references/utils/maps_docs.md | 8 + tests/documentation/test_docstring.py | 5 + 9 files changed, 219 insertions(+), 175 deletions(-) create mode 100644 docs/api_references/utils/filter.md create mode 100644 docs/api_references/utils/find.md create mode 100644 docs/api_references/utils/maps_docs.md diff --git a/docarray/typing/tensor/video/video_ndarray.py b/docarray/typing/tensor/video/video_ndarray.py index f6b3cd8d3b1..c237de5760e 100644 --- a/docarray/typing/tensor/video/video_ndarray.py +++ b/docarray/typing/tensor/video/video_ndarray.py @@ -42,15 +42,15 @@ class MyVideoDoc(BaseDoc): video_tensor=np.random.random((100, 224, 224, 3)), ) - doc_1.video_tensor.save(file_path='file_1.mp4') + doc_1.video_tensor.save(file_path='/tmp/file_1.mp4') doc_2 = MyVideoDoc( title='my_second_video_doc', - url='file_1.mp4', + url='/tmp/file_1.mp4', ) doc_2.video_tensor = parse_obj_as(VideoNdArray, doc_2.url.load().video) - doc_2.video_tensor.save(file_path='file_2.mp4') + doc_2.video_tensor.save(file_path='/tmp/file_2.mp4') ``` --- diff --git a/docarray/utils/filter.py b/docarray/utils/filter.py index a68641a7b13..773cbbe815d 100644 --- a/docarray/utils/filter.py +++ b/docarray/utils/filter.py @@ -1,3 +1,5 @@ +__all__ = ['filter_docs'] + import json from typing import Dict, List, Union @@ -13,50 +15,55 @@ def filter_docs( Filter the Documents in the index according to the given filter query. - EXAMPLE USAGE - .. code-block:: python + --- - from docarray import DocArray, BaseDoc - from docarray.documents import Text, Image - from docarray.util.filter import filter_docs + ```python + from docarray import DocArray, BaseDoc + from docarray.documents import TextDoc, ImageDoc + from docarray.utils.filter import filter_docs - class MyDocument(BaseDoc): - caption: Text - image: Image - price: int + class MyDocument(BaseDoc): + caption: TextDoc + ImageDoc: ImageDoc + price: int - docs = DocArray[MyDocument]( - [ - MyDocument( - caption='A tiger in the jungle', - image=Image(url='tigerphoto.png'), - price=100, - ), - MyDocument( - caption='A swimming turtle', image=Image(url='turtlepic.png'), price=50 - ), - MyDocument( - caption='A couple birdwatching with binoculars', - image=Image(url='binocularsphoto.png'), - price=30, - ), - ] - ) - query = { - '$and': { - 'image__url': {'$regex': 'photo'}, - 'price': {'$lte': 50}, - } + docs = DocArray[MyDocument]( + [ + MyDocument( + caption='A tiger in the jungle', + ImageDoc=ImageDoc(url='tigerphoto.png'), + price=100, + ), + MyDocument( + caption='A swimming turtle', + ImageDoc=ImageDoc(url='turtlepic.png'), + price=50, + ), + MyDocument( + caption='A couple birdwatching with binoculars', + ImageDoc=ImageDoc(url='binocularsphoto.png'), + price=30, + ), + ] + ) + query = { + '$and': { + 'ImageDoc__url': {'$regex': 'photo'}, + 'price': {'$lte': 50}, } + } + + results = filter_docs(docs, query) + assert len(results) == 1 + assert results[0].price == 30 + assert results[0].caption == 'A couple birdwatching with binoculars' + assert results[0].ImageDoc.url == 'binocularsphoto.png' + ``` - results = filter_docs(docs, query) - assert len(results) == 1 - assert results[0].price == 30 - assert results[0].caption == 'A couple birdwatching with binoculars' - assert results[0].image.url == 'binocularsphoto.png' + --- :param docs: the DocArray where to apply the filter :param query: the query to filter by diff --git a/docarray/utils/find.py b/docarray/utils/find.py index b7029578b56..c7eeb787159 100644 --- a/docarray/utils/find.py +++ b/docarray/utils/find.py @@ -1,3 +1,5 @@ +__all__ = ['find', 'find_batched'] + from typing import Any, Dict, List, NamedTuple, Optional, Type, Union, cast from typing_inspect import is_union_type @@ -34,52 +36,48 @@ def find( Find the closest Documents in the index to the query. Supports PyTorch and NumPy embeddings. - .. note:: - This utility function is likely to be removed once - Document Stores are available. - At that point, and in-memory Document Store will serve the same purpose - by exposing a .find() method. - - .. note:: - This is a simple implementation that assumes the same embedding field name for - both query and index, does not support nested search, and does not support - hybrid (multi-vector) search. These shortcoming will be addressed in future - versions. + !!! note + This is a simple implementation of exact search. If you need to do advance + search using approximate nearest neighbours search or hybrid search or + multi vector search please take a look at the [BaseDoc][docarray.base_doc.doc.BaseDoc] - EXAMPLE USAGE + --- - .. code-block:: python + ```python + from docarray import DocArray, BaseDoc + from docarray.typing import TorchTensor + from docarray.utils.find import find + import torch - from docarray import DocArray, BaseDoc - from docarray.typing import TorchTensor - from docarray.util.find import find + class MyDocument(BaseDoc): + embedding: TorchTensor - class MyDocument(BaseDoc): - embedding: TorchTensor + index = DocArray[MyDocument]( + [MyDocument(embedding=torch.rand(128)) for _ in range(100)] + ) - index = DocArray[MyDocument]( - [MyDocument(embedding=torch.rand(128)) for _ in range(100)] - ) + # use Document as query + query = MyDocument(embedding=torch.rand(128)) + top_matches, scores = find( + index=index, + query=query, + embedding_field='embedding', + metric='cosine_sim', + ) - # use Document as query - query = MyDocument(embedding=torch.rand(128)) - top_matches, scores = find( - index=index, - query=query, - embedding_field='tensor', - metric='cosine_sim', - ) + # use tensor as query + query = torch.rand(128) + top_matches, scores = find( + index=index, + query=query, + embedding_field='embedding', + metric='cosine_sim', + ) + ``` - # use tensor as query - query = torch.rand(128) - top_matches, scores = find( - index=index, - query=query, - embedding_field='tensor', - metric='cosine_sim', - ) + --- :param index: the index of Documents to search in :param query: the query to search for @@ -123,54 +121,51 @@ def find_batched( Find the closest Documents in the index to the queries. Supports PyTorch and NumPy embeddings. - .. note:: - This utility function is likely to be removed once - Document Stores are available. - At that point, and in-memory Document Store will serve the same purpose - by exposing a .find() method. - - .. note:: - This is a simple implementation that assumes the same embedding field name for - both query and index, does not support nested search, and does not support - hybrid (multi-vector) search. These shortcoming will be addressed in future - versions. - - EXAMPLE USAGE - - .. code-block:: python - - from docarray import DocArray, BaseDoc - from docarray.typing import TorchTensor - from docarray.util.find import find - - - class MyDocument(BaseDoc): - embedding: TorchTensor - - - index = DocArray[MyDocument]( - [MyDocument(embedding=torch.rand(128)) for _ in range(100)] - ) - - # use DocArray as query - query = DocArray[MyDocument]([MyDocument(embedding=torch.rand(128)) for _ in range(3)]) - results = find( - index=index, - query=query, - embedding_field='tensor', - metric='cosine_sim', - ) - top_matches, scores = results[0] - - # use tensor as query - query = torch.rand(3, 128) - results, scores = find( - index=index, - query=query, - embedding_field='tensor', - metric='cosine_sim', - ) - top_matches, scores = results[0] + !!! note + This is a simple implementation of exact search. If you need to do advance + search using approximate nearest neighbours search or hybrid search or + multi vector search please take a look at the [BaseDoc][docarray.base_doc.doc.BaseDoc] + + + --- + + ```python + # from docarray import DocArray, BaseDoc + # from docarray.typing import TorchTensor + # from docarray.utils.find import find + # import torch + # + # + # class MyDocument(BaseDoc): + # embedding: TorchTensor + # + # + # index = DocArray[MyDocument]( + # [MyDocument(embedding=torch.rand(128)) for _ in range(100)] + # ) + # + # # use DocArray as query + # query = DocArray[MyDocument]([MyDocument(embedding=torch.rand(128)) for _ in range(3)]) + # results = find( + # index=index, + # query=query, + # embedding_field='embedding', + # metric='cosine_sim', + # ) + # top_matches, scores = results[0] + # + # # use tensor as query + # query = torch.rand(3, 128) + # results, scores = find( + # index=index, + # query=query, + # embedding_field='embedding', + # metric='cosine_sim', + # ) + # top_matches, scores = results[0] + ``` + + --- :param index: the index of Documents to search in :param query: the query to search for diff --git a/docarray/utils/map.py b/docarray/utils/map.py index 9642d38f10a..3f8af0f2b7a 100644 --- a/docarray/utils/map.py +++ b/docarray/utils/map.py @@ -1,3 +1,4 @@ +__all__ = ['map_docs', 'map_docs_batch'] from contextlib import nullcontext from math import ceil from multiprocessing.pool import Pool, ThreadPool @@ -25,27 +26,33 @@ def map_docs( Return an iterator that applies `func` to every Document in `da` in parallel, yielding the results. - EXAMPLE USAGE - - .. code-block:: python - - from docarray import DocArray - from docarray.documents import Image - from docarray.utils.map import map_docs - - - def load_url_to_tensor(img: Image) -> Image: - img.tensor = img.url.load() - return img - - - da = DocArray[Image]([Image(url='/path/to/img.png') for _ in range(100)]) - da = DocArray[Image]( - list(map_docs(da, load_url_to_tensor, backend='thread')) - ) # threading is usually a good option for IO-bound tasks such as loading an image from url - - for doc in da: - assert doc.tensor is not None + --- + + ```python + # from docarray import DocArray + # from docarray.documents import ImageDoc + # from docarray.utils.map import map_docs + # + # + # def load_url_to_tensor(img: ImageDoc) -> ImageDoc: + # img.tensor = img.url.load() + # return img + # + # + # url = 'https://github.com/docarray/artwork/blob/main/stacked/color/docarray-stacked-color.png' + # + # + # da = DocArray[ImageDoc]([ImageDoc(url=url) for _ in range(100)]) + # da = DocArray[ImageDoc]( + # list(map_docs(da, load_url_to_tensor, backend='thread')) + # ) # threading is usually a good option for IO-bound tasks such as loading an + # # ImageDoc from url + # + # for doc in da: + # assert doc.tensor is not None + ``` + + --- :param da: DocArray to apply function to :param func: a function that takes a :class:`BaseDoc` as input and outputs @@ -72,7 +79,7 @@ def load_url_to_tensor(img: Image) -> Image: be responsible for closing the pool. :param show_progress: show a progress bar. Defaults to False. - :yield: Documents returned from `func` + :return: yield Documents returned from `func` """ if backend == 'process' and _is_lambda_or_partial_or_local_function(func): @@ -109,34 +116,39 @@ def map_docs_batch( yielding the results. Each element in the returned iterator is an :class:`AnyDocArray`. - EXAMPLE USAGE - - .. code-block:: python - from docarray import BaseDoc, DocArray - from docarray.utils.map import map_docs_batch - - - class MyDoc(BaseDoc): - name: str - - - def upper_case_name(da: DocArray[MyDoc]) -> DocArray[MyDoc]: - da.name = [n.upper() for n in da.name] - return da - - - batch_size = 16 - da = DocArray[MyDoc]([MyDoc(name='my orange cat') for _ in range(100)]) - it = map_docs_batch(da, upper_case_name, batch_size=batch_size) - for i, d in enumerate(it): - da[i * batch_size : (i + 1) * batch_size] = d - - assert len(da) == 100 - print(da.name[:3]) - - .. code-block:: text - - ['MY ORANGE CAT', 'MY ORANGE CAT', 'MY ORANGE CAT'] + --- + + ```python + # from docarray import BaseDoc, DocArray + # from docarray.utils.map import map_docs_batch + # + # + # class MyDoc(BaseDoc): + # name: str + # + # + # def upper_case_name(da: DocArray[MyDoc]) -> DocArray[MyDoc]: + # da.name = [n.upper() for n in da.name] + # return da + # + # + # batch_size = 16 + # da = DocArray[MyDoc]([MyDoc(name='my orange cat') for _ in range(100)]) + # it = map_docs_batch(da, upper_case_name, batch_size=batch_size) + # for i, d in enumerate(it): + # da[i * batch_size : (i + 1) * batch_size] = d + # + # assert len(da) == 100 + # print(da.name[:3]) + ``` + + --- + + ``` + ['MY ORANGE CAT', 'MY ORANGE CAT', 'MY ORANGE CAT'] + ``` + + --- :param da: DocArray to apply function to :param batch_size: Size of each generated batch (except the last one, which might @@ -166,7 +178,7 @@ def upper_case_name(da: DocArray[MyDoc]) -> DocArray[MyDoc]: :param pool: use an existing/external pool. If given, `backend` is ignored and you will be responsible for closing the pool. - :yield: DocArrays returned from `func` + :return: yield DocArrays returned from `func` """ if backend == 'process' and _is_lambda_or_partial_or_local_function(func): raise ValueError( diff --git a/docarray/utils/reduce.py b/docarray/utils/reduce.py index c021febea00..abf677b7cc9 100644 --- a/docarray/utils/reduce.py +++ b/docarray/utils/reduce.py @@ -1,3 +1,5 @@ +__all__ = ['reduce', 'reduce_all'] + from typing import Dict, List, Optional from docarray import DocArray diff --git a/docs/api_references/utils/filter.md b/docs/api_references/utils/filter.md new file mode 100644 index 00000000000..8eedc91509c --- /dev/null +++ b/docs/api_references/utils/filter.md @@ -0,0 +1,7 @@ +# Filter + +::: docarray.utils.filter.filter_docs + + + + diff --git a/docs/api_references/utils/find.md b/docs/api_references/utils/find.md new file mode 100644 index 00000000000..097e7372d29 --- /dev/null +++ b/docs/api_references/utils/find.md @@ -0,0 +1,8 @@ +# Find + +::: docarray.utils.find.find +::: docarray.utils.find.find_batched + + + + diff --git a/docs/api_references/utils/maps_docs.md b/docs/api_references/utils/maps_docs.md new file mode 100644 index 00000000000..3f663b114b3 --- /dev/null +++ b/docs/api_references/utils/maps_docs.md @@ -0,0 +1,8 @@ +# Map + +::: docarray.utils.map.map_docs +::: docarray.utils.map.map_docs_batch + + + + diff --git a/tests/documentation/test_docstring.py b/tests/documentation/test_docstring.py index 3d3cafe2311..4848601ff72 100644 --- a/tests/documentation/test_docstring.py +++ b/tests/documentation/test_docstring.py @@ -15,6 +15,7 @@ import docarray.index import docarray.store import docarray.typing +from docarray.utils import filter, find, map SUB_MODULE_TO_CHECK = [ docarray, @@ -23,6 +24,9 @@ docarray.documents, docarray.store, docarray.typing, + find, + map, + filter, ] @@ -38,6 +42,7 @@ def get_obj_to_check(lib): for lib in SUB_MODULE_TO_CHECK: obj_to_check.extend(get_obj_to_check(lib)) + members = [] for obj in obj_to_check: members.extend(get_codeblock_members(obj)) From e1cf96b636ae0c1fe42ca6376231bcf27b557402 Mon Sep 17 00:00:00 2001 From: Johannes Messner <44071807+JohannesMessner@users.noreply.github.com> Date: Wed, 29 Mar 2023 14:44:21 +0200 Subject: [PATCH 54/66] docs: fix docstring example of find_batched (#1308) Signed-off-by: Johannes Messner Signed-off-by: anna-charlotte --- docarray/utils/find.py | 66 +++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/docarray/utils/find.py b/docarray/utils/find.py index c7eeb787159..a626134d1b6 100644 --- a/docarray/utils/find.py +++ b/docarray/utils/find.py @@ -130,39 +130,39 @@ def find_batched( --- ```python - # from docarray import DocArray, BaseDoc - # from docarray.typing import TorchTensor - # from docarray.utils.find import find - # import torch - # - # - # class MyDocument(BaseDoc): - # embedding: TorchTensor - # - # - # index = DocArray[MyDocument]( - # [MyDocument(embedding=torch.rand(128)) for _ in range(100)] - # ) - # - # # use DocArray as query - # query = DocArray[MyDocument]([MyDocument(embedding=torch.rand(128)) for _ in range(3)]) - # results = find( - # index=index, - # query=query, - # embedding_field='embedding', - # metric='cosine_sim', - # ) - # top_matches, scores = results[0] - # - # # use tensor as query - # query = torch.rand(3, 128) - # results, scores = find( - # index=index, - # query=query, - # embedding_field='embedding', - # metric='cosine_sim', - # ) - # top_matches, scores = results[0] + from docarray import DocArray, BaseDoc + from docarray.typing import TorchTensor + from docarray.utils.find import find_batched + import torch + + + class MyDocument(BaseDoc): + embedding: TorchTensor + + + index = DocArray[MyDocument]( + [MyDocument(embedding=torch.rand(128)) for _ in range(100)] + ) + + # use DocArray as query + query = DocArray[MyDocument]([MyDocument(embedding=torch.rand(128)) for _ in range(3)]) + results = find_batched( + index=index, + query=query, + embedding_field='embedding', + metric='cosine_sim', + ) + top_matches, scores = results[0] + + # use tensor as query + query = torch.rand(3, 128) + results = find_batched( + index=index, + query=query, + embedding_field='embedding', + metric='cosine_sim', + ) + top_matches, scores = results[0] ``` --- From 301074119a2814c60508dbeb92733b7abf87eea8 Mon Sep 17 00:00:00 2001 From: samsja <55492238+samsja@users.noreply.github.com> Date: Wed, 29 Mar 2023 14:55:30 +0200 Subject: [PATCH 55/66] docs: fix map docstring (#1311) * fix: fix utils Signed-off-by: samsja * fix: fix map Signed-off-by: samsja --------- Signed-off-by: samsja Signed-off-by: anna-charlotte --- docarray/utils/map.py | 86 +++++++++++++------------- docs/api_references/utils/filter.md | 2 +- docs/api_references/utils/find.md | 2 +- docs/api_references/utils/maps_docs.md | 2 +- docs/api_references/utils/reduce.md | 8 +++ 5 files changed, 55 insertions(+), 45 deletions(-) create mode 100644 docs/api_references/utils/reduce.md diff --git a/docarray/utils/map.py b/docarray/utils/map.py index 3f8af0f2b7a..8ce4d974b18 100644 --- a/docarray/utils/map.py +++ b/docarray/utils/map.py @@ -29,27 +29,29 @@ def map_docs( --- ```python - # from docarray import DocArray - # from docarray.documents import ImageDoc - # from docarray.utils.map import map_docs - # - # - # def load_url_to_tensor(img: ImageDoc) -> ImageDoc: - # img.tensor = img.url.load() - # return img - # - # - # url = 'https://github.com/docarray/artwork/blob/main/stacked/color/docarray-stacked-color.png' - # - # - # da = DocArray[ImageDoc]([ImageDoc(url=url) for _ in range(100)]) - # da = DocArray[ImageDoc]( - # list(map_docs(da, load_url_to_tensor, backend='thread')) - # ) # threading is usually a good option for IO-bound tasks such as loading an - # # ImageDoc from url - # - # for doc in da: - # assert doc.tensor is not None + from docarray import DocArray + from docarray.documents import ImageDoc + from docarray.utils.map import map_docs + + + def load_url_to_tensor(img: ImageDoc) -> ImageDoc: + img.tensor = img.url.load() + return img + + + url = ( + 'https://upload.wikimedia.org/wikipedia/commons/8/80/' + 'Dag_Sebastian_Ahlander_at_G%C3%B6teborg_Book_Fair_2012b.jpg' + ) + + da = DocArray[ImageDoc]([ImageDoc(url=url) for _ in range(100)]) + da = DocArray[ImageDoc]( + list(map_docs(da, load_url_to_tensor, backend='thread')) + ) # threading is usually a good option for IO-bound tasks such as loading an + # ImageDoc from url + + for doc in da: + assert doc.tensor is not None ``` --- @@ -119,27 +121,27 @@ def map_docs_batch( --- ```python - # from docarray import BaseDoc, DocArray - # from docarray.utils.map import map_docs_batch - # - # - # class MyDoc(BaseDoc): - # name: str - # - # - # def upper_case_name(da: DocArray[MyDoc]) -> DocArray[MyDoc]: - # da.name = [n.upper() for n in da.name] - # return da - # - # - # batch_size = 16 - # da = DocArray[MyDoc]([MyDoc(name='my orange cat') for _ in range(100)]) - # it = map_docs_batch(da, upper_case_name, batch_size=batch_size) - # for i, d in enumerate(it): - # da[i * batch_size : (i + 1) * batch_size] = d - # - # assert len(da) == 100 - # print(da.name[:3]) + from docarray import BaseDoc, DocArray + from docarray.utils.map import map_docs_batch + + + class MyDoc(BaseDoc): + name: str + + + def upper_case_name(da: DocArray[MyDoc]) -> DocArray[MyDoc]: + da.name = [n.upper() for n in da.name] + return da + + + batch_size = 16 + da = DocArray[MyDoc]([MyDoc(name='my orange cat') for _ in range(100)]) + it = map_docs_batch(da, upper_case_name, batch_size=batch_size) + for i, d in enumerate(it): + da[i * batch_size : (i + 1) * batch_size] = d + + assert len(da) == 100 + print(da.name[:3]) ``` --- diff --git a/docs/api_references/utils/filter.md b/docs/api_references/utils/filter.md index 8eedc91509c..b40c59c510a 100644 --- a/docs/api_references/utils/filter.md +++ b/docs/api_references/utils/filter.md @@ -1,4 +1,4 @@ -# Filter +# filter ::: docarray.utils.filter.filter_docs diff --git a/docs/api_references/utils/find.md b/docs/api_references/utils/find.md index 097e7372d29..e94a9401149 100644 --- a/docs/api_references/utils/find.md +++ b/docs/api_references/utils/find.md @@ -1,4 +1,4 @@ -# Find +# find ::: docarray.utils.find.find ::: docarray.utils.find.find_batched diff --git a/docs/api_references/utils/maps_docs.md b/docs/api_references/utils/maps_docs.md index 3f663b114b3..9ccd971b1fd 100644 --- a/docs/api_references/utils/maps_docs.md +++ b/docs/api_references/utils/maps_docs.md @@ -1,4 +1,4 @@ -# Map +# map ::: docarray.utils.map.map_docs ::: docarray.utils.map.map_docs_batch diff --git a/docs/api_references/utils/reduce.md b/docs/api_references/utils/reduce.md new file mode 100644 index 00000000000..9b6db3eea02 --- /dev/null +++ b/docs/api_references/utils/reduce.md @@ -0,0 +1,8 @@ +# reduce + +::: docarray.utils.reduce.reduce +::: docarray.utils.reduce.reduce_all + + + + From 9da5624ac2b7164bcdb475526694acc3cbbf6abf Mon Sep 17 00:00:00 2001 From: Anne Yang Date: Wed, 29 Mar 2023 21:52:27 +0800 Subject: [PATCH 56/66] feat: elasticsearch document index (#1196) * feat: __init__ of ElasticDocumentIndex Signed-off-by: AnneY * feat: add index func Signed-off-by: AnneY * feat: get and del funcs Signed-off-by: AnneY * fix: init and index creation Signed-off-by: AnneY * fix: __init__ and _index Signed-off-by: AnneY * fix: _get_items Signed-off-by: AnneY * feat: add _find Signed-off-by: AnneY * feat: add filter text and their batch version Signed-off-by: AnneY * feat: store id and get nested doc Signed-off-by: AnneY * fix: vector cannot be all zero Signed-off-by: AnneY * fix: __getitem__ raise error Signed-off-by: AnneY * feat: support more python types Signed-off-by: AnneY * fix: mypy Signed-off-by: AnneY * test: elastic index tests Signed-off-by: AnneY * test: comment scripts before ci setup Signed-off-by: AnneY * chore: add elasticsearch dependency to poetry Signed-off-by: AnneY * test: elastic index ci setup Signed-off-by: AnneY * feat: add num_candidates to rumtime config Signed-off-by: AnneY * fix: let user pass index_settings Signed-off-by: AnneY * feat: degrade to v7 and add query builder Signed-off-by: AnneY * fix: remove elastic_transport Signed-off-by: AnneY * feat: minor features Signed-off-by: AnneY * refactor: style fix Signed-off-by: AnneY * fix: fix mypy Signed-off-by: AnneY * feat: add chunk size to runtime config Signed-off-by: AnneY * fix: chunk size Signed-off-by: AnneY * feat: add chunk_size to funcs Signed-off-by: AnneY * feat: rewrite elastic v7 query builder Signed-off-by: AnneY * fix: poetry Signed-off-by: AnneY * fix: db_type should be elastic types Signed-off-by: AnneY * fix: minor adjustment Signed-off-by: AnneY * refactor: rename elastic index files Signed-off-by: AnneY * refactor: remove comments Signed-off-by: AnneY * feat: rename, batch operations, etc Signed-off-by: AnneY * test: add test for persistency and col config Signed-off-by: AnneY * feat: support more field types and subclass Signed-off-by: AnneY * feat: support more python types Signed-off-by: AnneY * test: tf, tensor and more elastic field types Signed-off-by: AnneY * fix: elastic should be optional in toml Co-authored-by: Charlotte Gerhaher Signed-off-by: Anne Yang * refactor: rename class Signed-off-by: AnneY * fix: change Dict to Mapping Signed-off-by: AnneY * fix: add AbstractTensor Signed-off-by: AnneY * test: rename class and add tests Signed-off-by: AnneY * fix: poetry Signed-off-by: AnneY --------- Signed-off-by: AnneY Signed-off-by: Anne Yang Co-authored-by: Charlotte Gerhaher --- docarray/index/abstract.py | 7 +- docarray/index/backends/elastic.py | 546 +++++++++++++++++++ docarray/index/backends/hnswlib.py | 6 +- poetry.lock | 25 +- pyproject.toml | 2 + tests/index/elastic/fixture.py | 58 ++ tests/index/elastic/v7/docker-compose.yml | 16 + tests/index/elastic/v7/test_column_config.py | 128 +++++ tests/index/elastic/v7/test_find.py | 318 +++++++++++ tests/index/elastic/v7/test_index_get_del.py | 280 ++++++++++ 10 files changed, 1379 insertions(+), 7 deletions(-) create mode 100644 docarray/index/backends/elastic.py create mode 100644 tests/index/elastic/fixture.py create mode 100644 tests/index/elastic/v7/docker-compose.yml create mode 100644 tests/index/elastic/v7/test_column_config.py create mode 100644 tests/index/elastic/v7/test_find.py create mode 100644 tests/index/elastic/v7/test_index_get_del.py diff --git a/docarray/index/abstract.py b/docarray/index/abstract.py index 89d8573bd23..7c12c42ea88 100644 --- a/docarray/index/abstract.py +++ b/docarray/index/abstract.py @@ -9,6 +9,7 @@ Generic, Iterable, List, + Mapping, NamedTuple, Optional, Sequence, @@ -234,13 +235,13 @@ def _find( @abstractmethod def _find_batched( self, - query: np.ndarray, + queries: np.ndarray, limit: int, search_field: str = '', ) -> _FindResultBatched: """Find documents in the index - :param query: query vectors for KNN/ANN search. + :param queries: query vectors for KNN/ANN search. Has shape (batch_size, vector_dim) :param limit: maximum number of documents to return :param search_field: name of the field to search on @@ -594,7 +595,7 @@ def _get_values_by_column(docs: Sequence[BaseDoc], col_name: str) -> List[Any]: @staticmethod def _transpose_col_value_dict( - col_value_dict: Dict[str, Iterable[Any]] + col_value_dict: Mapping[str, Iterable[Any]] ) -> Generator[Dict[str, Any], None, None]: """'Transpose' the output of `_get_col_value_dict()`: Yield rows of columns, where each row represent one Document. Since a generator is returned, this process comes at negligible cost. diff --git a/docarray/index/backends/elastic.py b/docarray/index/backends/elastic.py new file mode 100644 index 00000000000..deefc3b2a86 --- /dev/null +++ b/docarray/index/backends/elastic.py @@ -0,0 +1,546 @@ +import os +import uuid +import warnings +from collections import defaultdict +from dataclasses import dataclass, field +from typing import ( + Any, + Dict, + Generator, + Generic, + Iterable, + List, + Mapping, + Optional, + Sequence, + Tuple, + Type, + TypeVar, + Union, + cast, +) + +import numpy as np +from elasticsearch import Elasticsearch +from elasticsearch.helpers import parallel_bulk +from pydantic import parse_obj_as + +import docarray.typing +from docarray import BaseDoc +from docarray.index.abstract import ( + BaseDocIndex, + _ColumnInfo, + _FindResultBatched, + _raise_not_composable, +) +from docarray.typing import AnyTensor +from docarray.typing.tensor.abstract_tensor import AbstractTensor +from docarray.typing.tensor.ndarray import NdArray +from docarray.utils._internal.misc import is_tf_available, is_torch_available +from docarray.utils.find import _FindResult + +TSchema = TypeVar('TSchema', bound=BaseDoc) +T = TypeVar('T', bound='ElasticV7DocIndex') + +ELASTIC_PY_VEC_TYPES: List[Any] = [list, tuple, np.ndarray, AbstractTensor] + +if is_torch_available(): + import torch + + ELASTIC_PY_VEC_TYPES.append(torch.Tensor) + +if is_tf_available(): + import tensorflow as tf # type: ignore + + from docarray.typing import TensorFlowTensor + + ELASTIC_PY_VEC_TYPES.append(tf.Tensor) + ELASTIC_PY_VEC_TYPES.append(TensorFlowTensor) + + +class ElasticV7DocIndex(BaseDocIndex, Generic[TSchema]): + def __init__(self, db_config=None, **kwargs): + super().__init__(db_config=db_config, **kwargs) + self._db_config = cast(ElasticV7DocIndex.DBConfig, self._db_config) + + if self._db_config.index_name is None: + id = uuid.uuid4().hex + self._db_config.index_name = 'index__' + id + + self._index_name = self._db_config.index_name + + self._client = Elasticsearch( + hosts=self._db_config.hosts, + **self._db_config.es_config, + ) + + # compatibility + self._server_version = self._client.info()['version']['number'] + if int(self._server_version.split('.')[0]) >= 8: + os.environ['ELASTIC_CLIENT_APIVERSIONING'] = '1' + + body: Dict[str, Any] = { + 'mappings': { + 'dynamic': True, + '_source': {'enabled': 'true'}, + 'properties': {}, + } + } + + for col_name, col in self._column_infos.items(): + body['mappings']['properties'][col_name] = self._create_index_mapping(col) + + if self._client.indices.exists(index=self._index_name): + self._client.indices.put_mapping( + index=self._index_name, body=body['mappings'] + ) + else: + self._client.indices.create(index=self._index_name, body=body) + + if len(self._db_config.index_settings): + self._client.indices.put_settings( + index=self._index_name, body=self._db_config.index_settings + ) + + self._refresh(self._index_name) + + ############################################### + # Inner classes for query builder and configs # + ############################################### + + class QueryBuilder(BaseDocIndex.QueryBuilder): + def __init__(self, outer_instance, **kwargs): + super().__init__() + self._outer_instance = outer_instance + self._query: Dict[str, Any] = { + 'query': defaultdict(lambda: defaultdict(list)) + } + + def build(self, *args, **kwargs) -> Any: + if ( + 'script_score' in self._query['query'] + and 'bool' in self._query['query'] + and len(self._query['query']['bool']) > 0 + ): + self._query['query']['script_score']['query'] = {} + self._query['query']['script_score']['query']['bool'] = self._query[ + 'query' + ]['bool'] + del self._query['query']['bool'] + + return self._query + + def find( + self, + query: Union[AnyTensor, BaseDoc], + search_field: str = 'embedding', + limit: int = 10, + ): + if isinstance(query, BaseDoc): + query_vec = BaseDocIndex._get_values_by_column([query], search_field)[0] + else: + query_vec = query + query_vec_np = BaseDocIndex._to_numpy(self._outer_instance, query_vec) + self._query['size'] = limit + self._query['query']['script_score'] = ElasticV7DocIndex._form_search_body( + query_vec_np, limit, search_field + )['query']['script_score'] + + return self + + def filter(self, query: Dict[str, Any], limit: int = 10): + self._query['size'] = limit + self._query['query']['bool']['filter'].append(query) + return self + + def text_search(self, query: str, search_field: str = 'text', limit: int = 10): + self._query['size'] = limit + self._query['query']['bool']['must'].append( + {'match': {search_field: query}} + ) + return self + + find_batched = _raise_not_composable('find_batched') + filter_batched = _raise_not_composable('find_batched') + text_search_batched = _raise_not_composable('text_search') + + def build_query(self, **kwargs) -> QueryBuilder: + """ + Build a query for this DocumentIndex. + """ + return self.QueryBuilder(self, **kwargs) + + @dataclass + class DBConfig(BaseDocIndex.DBConfig): + hosts: Union[str, List[str], None] = 'http://localhost:9200' + index_name: Optional[str] = None + es_config: Dict[str, Any] = field(default_factory=dict) + index_settings: Dict[str, Any] = field(default_factory=dict) + + @dataclass + class RuntimeConfig(BaseDocIndex.RuntimeConfig): + default_column_config: Dict[Any, Dict[str, Any]] = field( + default_factory=lambda: { + 'binary': {}, + 'boolean': {}, + 'keyword': {}, + 'long': {}, + 'integer': {}, + 'short': {}, + 'byte': {}, + 'double': {}, + 'float': {}, + 'half_float': {}, + 'scaled_float': {}, + 'unsigned_long': {}, + 'dates': {}, + 'alias': {}, + 'object': {}, + 'flattened': {}, + 'nested': {}, + 'join': {}, + 'integer_range': {}, + 'float_range': {}, + 'long_range': {}, + 'double_range': {}, + 'date_range': {}, + 'ip': {}, + 'version': {}, + 'histogram': {}, + 'text': {}, + 'annotated_text': {}, + 'completion': {}, + 'search_as_you_type': {}, + 'token_count': {}, + 'dense_vector': {'dims': 128}, + 'sparse_vector': {}, + 'rank_feature': {}, + 'rank_features': {}, + 'geo_point': {}, + 'geo_shape': {}, + 'point': {}, + 'shape': {}, + 'percolator': {}, + # `None` is not a Type, but we allow it here anyway + None: {}, # type: ignore + } + ) + chunk_size: int = 500 + + ############################################### + # Implementation of abstract methods # + ############################################### + + def python_type_to_db_type(self, python_type: Type) -> Any: + """Map python type to database type.""" + + for allowed_type in ELASTIC_PY_VEC_TYPES: + if issubclass(python_type, allowed_type): + return 'dense_vector' + + elastic_py_types = { + docarray.typing.ID: 'keyword', + docarray.typing.AnyUrl: 'keyword', + bool: 'boolean', + int: 'integer', + float: 'float', + str: 'text', + bytes: 'binary', + dict: 'object', + } + + for type in elastic_py_types.keys(): + if issubclass(python_type, type): + return elastic_py_types[type] + + raise ValueError(f'Unsupported column type for {type(self)}: {python_type}') + + def _index( + self, + column_to_data: Mapping[str, Generator[Any, None, None]], + refresh: bool = True, + chunk_size: Optional[int] = None, + ): + + data = self._transpose_col_value_dict(column_to_data) + requests = [] + + for row in data: + request = { + '_index': self._index_name, + '_id': row['id'], + } + for col_name, col in self._column_infos.items(): + if col.db_type == 'dense_vector' and np.all(row[col_name] == 0): + row[col_name] = row[col_name] + 1.0e-9 + if row[col_name] is None: + continue + request[col_name] = row[col_name] + requests.append(request) + + _, warning_info = self._send_requests(requests, chunk_size) + for info in warning_info: + warnings.warn(str(info)) + + if refresh: + self._refresh(self._index_name) + + def num_docs(self) -> int: + return self._client.count(index=self._index_name)['count'] + + def _del_items( + self, + doc_ids: Sequence[str], + chunk_size: Optional[int] = None, + ): + requests = [] + for _id in doc_ids: + requests.append( + {'_op_type': 'delete', '_index': self._index_name, '_id': _id} + ) + + _, warning_info = self._send_requests(requests, chunk_size) + + # raise warning if some ids are not found + if warning_info: + ids = [info['delete']['_id'] for info in warning_info] + warnings.warn(f'No document with id {ids} found') + + self._refresh(self._index_name) + + def _get_items(self, doc_ids: Sequence[str]) -> Sequence[TSchema]: + accumulated_docs = [] + accumulated_docs_id_not_found = [] + + es_rows = self._client.mget( + index=self._index_name, + body={'ids': doc_ids}, + )['docs'] + + for row in es_rows: + if row['found']: + doc_dict = row['_source'] + accumulated_docs.append(doc_dict) + else: + accumulated_docs_id_not_found.append(row['_id']) + + # raise warning if some ids are not found + if accumulated_docs_id_not_found: + warnings.warn(f'No document with id {accumulated_docs_id_not_found} found') + + return accumulated_docs + + def execute_query(self, query: Dict[str, Any], *args, **kwargs) -> Any: + if args or kwargs: + raise ValueError( + f'args and kwargs not supported for `execute_query` on {type(self)}' + ) + + resp = self._client.search(index=self._index_name, body=query) + docs, scores = self._format_response(resp) + + return _FindResult(documents=docs, scores=scores) + + def _find( + self, query: np.ndarray, limit: int, search_field: str = '' + ) -> _FindResult: + if int(self._server_version.split('.')[0]) >= 8: + warnings.warn( + 'You are using Elasticsearch 8.0+ and the current client is 7.10.1. HNSW based vector search is not supported and the find method has a default implementation using exhaustive KNN search with cosineSimilarity, which may result in slow performance.' + ) + + body = self._form_search_body(query, limit, search_field) + + resp = self._client.search( + index=self._index_name, + body=body, + ) + + docs, scores = self._format_response(resp) + + return _FindResult(documents=docs, scores=scores) + + def _find_batched( + self, + queries: np.ndarray, + limit: int, + search_field: str = '', + ) -> _FindResultBatched: + request = [] + for query in queries: + head = {'index': self._index_name} + body = self._form_search_body(query, limit, search_field) + request.extend([head, body]) + + responses = self._client.msearch(body=request) + + das, scores = zip( + *[self._format_response(resp) for resp in responses['responses']] + ) + return _FindResultBatched(documents=list(das), scores=np.array(scores)) + + def _filter( + self, + filter_query: Dict[str, Any], + limit: int, + ) -> List[Dict]: + body = { + 'size': limit, + 'query': filter_query, + } + + resp = self._client.search( + index=self._index_name, + body=body, + ) + + docs, _ = self._format_response(resp) + + return docs + + def _filter_batched( + self, + filter_queries: Any, + limit: int, + ) -> List[List[Dict]]: + request = [] + for query in filter_queries: + head = {'index': self._index_name} + body = {'query': query, 'size': limit} + request.extend([head, body]) + + responses = self._client.msearch(body=request) + das, _ = zip(*[self._format_response(resp) for resp in responses['responses']]) + + return list(das) + + def _text_search( + self, + query: str, + limit: int, + search_field: str = '', + ) -> _FindResult: + + body = self._form_text_search_body(query, limit, search_field) + + resp = self._client.search( + index=self._index_name, + body=body, + ) + + docs, scores = self._format_response(resp) + + return _FindResult(documents=docs, scores=scores) + + def _text_search_batched( + self, + queries: Sequence[str], + limit: int, + search_field: str = '', + ) -> _FindResultBatched: + request = [] + for query in queries: + head = {'index': self._index_name} + body = self._form_text_search_body(query, limit, search_field) + request.extend([head, body]) + + responses = self._client.msearch(body=request) + + das, scores = zip( + *[self._format_response(resp) for resp in responses['responses']] + ) + return _FindResultBatched(documents=list(das), scores=np.array(scores)) + + ############################################### + # Helpers # + ############################################### + + # ElasticSearch helpers + def _create_index_mapping(self, col: '_ColumnInfo') -> Dict[str, Any]: + """Create a new HNSW index for a column, and initialize it.""" + + index = col.config.copy() + if 'type' not in index: + index['type'] = col.db_type + + if col.db_type == 'dense_vector' and col.n_dim: + index['dims'] = col.n_dim + + return index + + def _send_requests( + self, + request: Iterable[Dict[str, Any]], + chunk_size: Optional[int] = None, + **kwargs, + ) -> Tuple[List[Dict], List[Any]]: + """Send bulk request to Elastic and gather the successful info""" + + accumulated_info = [] + warning_info = [] + for success, info in parallel_bulk( + self._client, + request, + raise_on_error=False, + raise_on_exception=False, + chunk_size=chunk_size if chunk_size else self._runtime_config.chunk_size, # type: ignore + **kwargs, + ): + if not success: + warning_info.append(info) + else: + accumulated_info.append(info) + + return accumulated_info, warning_info + + @staticmethod + def _form_search_body( + query: np.ndarray, limit: int, search_field: str = '' + ) -> Dict[str, Any]: + body = { + 'size': limit, + 'query': { + 'script_score': { + 'query': {'match_all': {}}, + 'script': { + 'source': f'cosineSimilarity(params.query_vector, \'{search_field}\') + 1.0', + 'params': {'query_vector': query}, + }, + } + }, + } + return body + + @staticmethod + def _form_text_search_body( + query: str, limit: int, search_field: str = '' + ) -> Dict[str, Any]: + body = { + 'size': limit, + 'query': { + 'bool': { + 'must': {'match': {search_field: query}}, + } + }, + } + return body + + def _format_response(self, response: Any) -> Tuple[List[Dict], NdArray]: + docs = [] + scores = [] + for result in response['hits']['hits']: + if not isinstance(result, dict): + result = result.to_dict() + + if result.get('_source', None): + doc_dict = result['_source'] + else: + doc_dict = result['fields'] + doc_dict['id'] = result['_id'] + docs.append(doc_dict) + scores.append(result['_score']) + + return docs, parse_obj_as(NdArray, scores) + + def _refresh(self, index_name: str): + self._client.indices.refresh(index=index_name) diff --git a/docarray/index/backends/hnswlib.py b/docarray/index/backends/hnswlib.py index c234d271c3c..b7555012db8 100644 --- a/docarray/index/backends/hnswlib.py +++ b/docarray/index/backends/hnswlib.py @@ -241,12 +241,12 @@ def execute_query(self, query: List[Tuple[str, Dict]], *args, **kwargs) -> Any: def _find_batched( self, - query: np.ndarray, + queries: np.ndarray, limit: int, search_field: str = '', ) -> _FindResultBatched: index = self._hnsw_indices[search_field] - labels, distances = index.knn_query(query, k=limit) + labels, distances = index.knn_query(queries, k=limit) result_das = [ self._get_docs_sqlite_hashed_id( ids_per_query.tolist(), @@ -260,7 +260,7 @@ def _find( ) -> _FindResult: query_batched = np.expand_dims(query, axis=0) docs, scores = self._find_batched( - query=query_batched, limit=limit, search_field=search_field + queries=query_batched, limit=limit, search_field=search_field ) return _FindResult(documents=docs[0], scores=scores[0]) diff --git a/poetry.lock b/poetry.lock index 473da526ef1..a9bc680af7f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -803,6 +803,28 @@ six = ">=1.9.0" gmpy = ["gmpy"] gmpy2 = ["gmpy2"] +[[package]] +name = "elasticsearch" +version = "7.10.1" +description = "Python client for Elasticsearch" +category = "main" +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, <4" +files = [ + {file = "elasticsearch-7.10.1-py2.py3-none-any.whl", hash = "sha256:4ebd34fd223b31c99d9f3b6b6236d3ac18b3046191a37231e8235b06ae7db955"}, + {file = "elasticsearch-7.10.1.tar.gz", hash = "sha256:a725dd923d349ca0652cf95d6ce23d952e2153740cf4ab6daf4a2d804feeed48"}, +] + +[package.dependencies] +certifi = "*" +urllib3 = ">=1.21.1,<2" + +[package.extras] +async = ["aiohttp (>=3,<4)"] +develop = ["black", "coverage", "jinja2", "mock", "pytest", "pytest-cov", "pyyaml", "requests (>=2.0.0,<3.0.0)", "sphinx (<1.7)", "sphinx-rtd-theme"] +docs = ["sphinx (<1.7)", "sphinx-rtd-theme"] +requests = ["requests (>=2.4.0,<3.0.0)"] + [[package]] name = "entrypoints" version = "0.4" @@ -4569,6 +4591,7 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" audio = ["pydub"] aws = ["smart-open"] common = ["protobuf", "lz4"] +elasticsearch = ["elasticsearch"] full = ["protobuf", "lz4", "pandas", "pillow", "types-pillow", "av", "pydub", "trimesh"] hnswlib = ["hnswlib"] image = ["pillow", "types-pillow"] @@ -4582,4 +4605,4 @@ web = ["fastapi"] [metadata] lock-version = "2.0" python-versions = ">=3.7,<4.0" -content-hash = "33f01d36bd1b134d7ab90505bba21a95a500d0898aeaae63e71e8213616098c2" +content-hash = "821f6cd00f78c456f6146f39c14f0704e4f2d113c35db00c58462d8cfbe3a538" diff --git a/pyproject.toml b/pyproject.toml index f8accde29e3..3114ff8dc61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ hnswlib = {version = ">=0.6.2", optional = true } lz4 = {version= ">=1.0.0", optional = true} pydub = {version = "^0.25.1", optional = true } pandas = {version = ">=1.1.0", optional = true } +elasticsearch = {version = "7.10.1", optional = true } smart-open = {version = ">=6.3.0", extras = ["s3"], optional = true} jina-hubble-sdk = {version = ">=0.34.0", optional = true} @@ -35,6 +36,7 @@ video = ["av"] audio = ["pydub"] mesh = ["trimesh"] hnswlib = ["hnswlib"] +elasticsearch = ["elasticsearch"] jac = ["jina-hubble-sdk"] aws = ["smart-open"] torch = ["torch"] diff --git a/tests/index/elastic/fixture.py b/tests/index/elastic/fixture.py new file mode 100644 index 00000000000..1caa31da2a6 --- /dev/null +++ b/tests/index/elastic/fixture.py @@ -0,0 +1,58 @@ +import os +import time + +import pytest +from pydantic import Field + +from docarray import BaseDoc +from docarray.typing import NdArray + +pytestmark = [pytest.mark.slow, pytest.mark.doc_index] + + +class SimpleDoc(BaseDoc): + tens: NdArray[10] = Field(dims=1000) + + +class FlatDoc(BaseDoc): + tens_one: NdArray = Field(dims=10) + tens_two: NdArray = Field(dims=50) + + +class NestedDoc(BaseDoc): + d: SimpleDoc + + +class DeepNestedDoc(BaseDoc): + d: NestedDoc + + +cur_dir = os.path.dirname(os.path.abspath(__file__)) +compose_yml_v7 = os.path.abspath(os.path.join(cur_dir, 'v7/docker-compose.yml')) +compose_yml_v8 = os.path.abspath(os.path.join(cur_dir, 'v8/docker-compose.yml')) + + +@pytest.fixture(scope='module', autouse=True) +def start_storage_v7(): + os.system(f"docker-compose -f {compose_yml_v7} up -d --remove-orphans") + _wait_for_es() + + yield + os.system(f"docker-compose -f {compose_yml_v7} down --remove-orphans") + + +@pytest.fixture(scope='module', autouse=True) +def start_storage_v8(): + os.system(f"docker-compose -f {compose_yml_v8} up -d --remove-orphans") + _wait_for_es() + + yield + os.system(f"docker-compose -f {compose_yml_v8} down --remove-orphans") + + +def _wait_for_es(): + from elasticsearch import Elasticsearch + + es = Elasticsearch(hosts='http://localhost:9200/') + while not es.ping(): + time.sleep(0.5) diff --git a/tests/index/elastic/v7/docker-compose.yml b/tests/index/elastic/v7/docker-compose.yml new file mode 100644 index 00000000000..f4dd8a49d0b --- /dev/null +++ b/tests/index/elastic/v7/docker-compose.yml @@ -0,0 +1,16 @@ +version: "3.3" +services: + elastic: + image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2 + environment: + - xpack.security.enabled=false + - discovery.type=single-node + - ES_JAVA_OPTS=-Xmx1024m + ports: + - "9200:9200" + networks: + - elastic + +networks: + elastic: + name: elastic \ No newline at end of file diff --git a/tests/index/elastic/v7/test_column_config.py b/tests/index/elastic/v7/test_column_config.py new file mode 100644 index 00000000000..80f42c6a347 --- /dev/null +++ b/tests/index/elastic/v7/test_column_config.py @@ -0,0 +1,128 @@ +from pydantic import Field + +from docarray import BaseDoc +from docarray.index import ElasticV7DocIndex +from tests.index.elastic.fixture import start_storage_v7 # noqa: F401 + + +def test_column_config(): + class MyDoc(BaseDoc): + text: str + color: str = Field(col_type='keyword') + + store = ElasticV7DocIndex[MyDoc]() + index_docs = [ + MyDoc(id='0', text='hello world', color='red'), + MyDoc(id='1', text='never gonna give you up', color='blue'), + MyDoc(id='2', text='we are the world', color='green'), + ] + store.index(index_docs) + + query = 'world' + docs, _ = store.text_search(query, search_field='text') + assert [doc.id for doc in docs] == ['0', '2'] + + filter_query = {'terms': {'color': ['red', 'blue']}} + docs = store.filter(filter_query) + assert [doc.id for doc in docs] == ['0', '1'] + + +def test_field_object(): + class MyDoc(BaseDoc): + manager: dict = Field( + properties={ + 'age': {'type': 'integer'}, + 'name': { + 'properties': { + 'first': {'type': 'keyword'}, + 'last': {'type': 'keyword'}, + } + }, + } + ) + + store = ElasticV7DocIndex[MyDoc]() + doc = [ + MyDoc(manager={'age': 25, 'name': {'first': 'Rachel', 'last': 'Green'}}), + MyDoc(manager={'age': 30, 'name': {'first': 'Monica', 'last': 'Geller'}}), + MyDoc(manager={'age': 35, 'name': {'first': 'Phoebe', 'last': 'Buffay'}}), + ] + store.index(doc) + id_ = doc[0].id + assert store[id_].id == id_ + assert store[id_].manager == doc[0].manager + + filter_query = {'range': {'manager.age': {'gte': 30}}} + docs = store.filter(filter_query) + assert [doc.id for doc in docs] == [doc[1].id, doc[2].id] + + +def test_field_geo_point(): + class MyDoc(BaseDoc): + location: dict = Field(col_type='geo_point') + + store = ElasticV7DocIndex[MyDoc]() + doc = [ + MyDoc(location={'lat': 40.12, 'lon': -72.34}), + MyDoc(location={'lat': 41.12, 'lon': -73.34}), + MyDoc(location={'lat': 42.12, 'lon': -74.34}), + ] + store.index(doc) + + query = { + 'query': { + 'geo_bounding_box': { + 'location': { + 'top_left': {'lat': 42, 'lon': -74}, + 'bottom_right': {'lat': 40, 'lon': -72}, + } + } + }, + } + + docs, _ = store.execute_query(query) + assert [doc['id'] for doc in docs] == [doc[0].id, doc[1].id] + + +def test_field_range(): + class MyDoc(BaseDoc): + expected_attendees: dict = Field(col_type='integer_range') + time_frame: dict = Field(col_type='date_range', format='yyyy-MM-dd') + + store = ElasticV7DocIndex[MyDoc]() + doc = [ + MyDoc( + expected_attendees={'gte': 10, 'lt': 20}, + time_frame={'gte': '2023-01-01', 'lt': '2023-02-01'}, + ), + MyDoc( + expected_attendees={'gte': 20, 'lt': 30}, + time_frame={'gte': '2023-02-01', 'lt': '2023-03-01'}, + ), + MyDoc( + expected_attendees={'gte': 30, 'lt': 40}, + time_frame={'gte': '2023-03-01', 'lt': '2023-04-01'}, + ), + ] + store.index(doc) + + query = { + 'query': { + 'bool': { + 'should': [ + {'term': {'expected_attendees': {'value': 15}}}, + { + 'range': { + 'time_frame': { + 'gte': '2023-02-05', + 'lt': '2023-02-10', + 'relation': 'contains', + } + } + }, + ] + } + }, + } + docs, _ = store.execute_query(query) + assert [doc['id'] for doc in docs] == [doc[0].id, doc[1].id] diff --git a/tests/index/elastic/v7/test_find.py b/tests/index/elastic/v7/test_find.py new file mode 100644 index 00000000000..562b6f68326 --- /dev/null +++ b/tests/index/elastic/v7/test_find.py @@ -0,0 +1,318 @@ +import numpy as np +import torch +from pydantic import Field + +from docarray import BaseDoc +from docarray.index import ElasticV7DocIndex +from docarray.typing import NdArray, TorchTensor +from tests.index.elastic.fixture import start_storage_v7 # noqa: F401 +from tests.index.elastic.fixture import FlatDoc, SimpleDoc + + +def test_find_simple_schema(): + class SimpleSchema(BaseDoc): + tens: NdArray[10] + + store = ElasticV7DocIndex[SimpleSchema]() + + index_docs = [SimpleDoc(tens=np.random.rand(10)) for _ in range(10)] + store.index(index_docs) + + query = index_docs[-1] + docs, scores = store.find(query, search_field='tens', limit=5) + + assert len(docs) == 5 + assert len(scores) == 5 + + assert docs[0].id == index_docs[-1].id + assert np.allclose(docs[0].tens, index_docs[-1].tens) + + +def test_find_flat_schema(): + class FlatSchema(BaseDoc): + tens_one: NdArray = Field(dims=10) + tens_two: NdArray = Field(dims=50) + + store = ElasticV7DocIndex[FlatSchema]() + + index_docs = [ + FlatDoc(tens_one=np.random.rand(10), tens_two=np.random.rand(50)) + for _ in range(10) + ] + store.index(index_docs) + + query = index_docs[-1] + + # find on tens_one + docs, scores = store.find(query, search_field='tens_one', limit=5) + assert len(docs) == 5 + assert len(scores) == 5 + assert docs[0].id == index_docs[-1].id + assert np.allclose(docs[0].tens_one, index_docs[-1].tens_one) + assert np.allclose(docs[0].tens_two, index_docs[-1].tens_two) + + # find on tens_two + docs, scores = store.find(query, search_field='tens_two', limit=5) + assert len(docs) == 5 + assert len(scores) == 5 + assert docs[0].id == index_docs[-1].id + assert np.allclose(docs[0].tens_one, index_docs[-1].tens_one) + assert np.allclose(docs[0].tens_two, index_docs[-1].tens_two) + + +def test_find_nested_schema(): + class SimpleDoc(BaseDoc): + tens: NdArray[10] + + class NestedDoc(BaseDoc): + d: SimpleDoc + tens: NdArray[10] + + class DeepNestedDoc(BaseDoc): + d: NestedDoc + tens: NdArray = Field(dims=10) + + store = ElasticV7DocIndex[DeepNestedDoc]() + + index_docs = [ + DeepNestedDoc( + d=NestedDoc(d=SimpleDoc(tens=np.random.rand(10)), tens=np.random.rand(10)), + tens=np.random.rand(10), + ) + for _ in range(10) + ] + store.index(index_docs) + + query = index_docs[-1] + + # find on root level + docs, scores = store.find(query, search_field='tens', limit=5) + assert len(docs) == 5 + assert len(scores) == 5 + assert docs[0].id == index_docs[-1].id + assert np.allclose(docs[0].tens, index_docs[-1].tens) + + # find on first nesting level + docs, scores = store.find(query, search_field='d__tens', limit=5) + assert len(docs) == 5 + assert len(scores) == 5 + assert docs[0].id == index_docs[-1].id + assert np.allclose(docs[0].d.tens, index_docs[-1].d.tens) + + # find on second nesting level + docs, scores = store.find(query, search_field='d__d__tens', limit=5) + assert len(docs) == 5 + assert len(scores) == 5 + assert docs[0].id == index_docs[-1].id + assert np.allclose(docs[0].d.d.tens, index_docs[-1].d.d.tens) + + +def test_find_torch(): + class TorchDoc(BaseDoc): + tens: TorchTensor[10] + + store = ElasticV7DocIndex[TorchDoc]() + + # A dense_vector field stores dense vectors of float values. + index_docs = [ + TorchDoc(tens=np.random.rand(10).astype(dtype=np.float32)) for _ in range(10) + ] + store.index(index_docs) + + for doc in index_docs: + assert isinstance(doc.tens, TorchTensor) + + query = index_docs[-1] + docs, scores = store.find(query, search_field='tens', limit=5) + + assert len(docs) == 5 + assert len(scores) == 5 + for doc in docs: + assert isinstance(doc.tens, TorchTensor) + + assert docs[0].id == index_docs[-1].id + assert torch.allclose(docs[0].tens, index_docs[-1].tens) + + +def test_find_tensorflow(): + from docarray.typing import TensorFlowTensor + + class TfDoc(BaseDoc): + tens: TensorFlowTensor[10] + + store = ElasticV7DocIndex[TfDoc]() + + index_docs = [ + TfDoc(tens=np.random.rand(10).astype(dtype=np.float32)) for _ in range(10) + ] + store.index(index_docs) + + for doc in index_docs: + assert isinstance(doc.tens, TensorFlowTensor) + + query = index_docs[-1] + docs, scores = store.find(query, search_field='tens', limit=5) + + assert len(docs) == 5 + assert len(scores) == 5 + for doc in docs: + assert isinstance(doc.tens, TensorFlowTensor) + + assert docs[0].id == index_docs[-1].id + assert np.allclose( + docs[0].tens.unwrap().numpy(), index_docs[-1].tens.unwrap().numpy() + ) + + +def test_find_batched(): + store = ElasticV7DocIndex[SimpleDoc]() + + index_docs = [SimpleDoc(tens=np.random.rand(10)) for _ in range(10)] + store.index(index_docs) + + queries = index_docs[-2:] + docs_batched, scores_batched = store.find_batched( + queries, search_field='tens', limit=5 + ) + + for docs, scores, query in zip(docs_batched, scores_batched, queries): + assert len(docs) == 5 + assert len(scores) == 5 + assert docs[0].id == query.id + assert np.allclose(docs[0].tens, query.tens) + + +def test_filter(): + class MyDoc(BaseDoc): + A: bool + B: int + C: float + + store = ElasticV7DocIndex[MyDoc]() + + index_docs = [MyDoc(id=f'{i}', A=(i % 2 == 0), B=i, C=i + 0.5) for i in range(10)] + store.index(index_docs) + + filter_query = {'term': {'A': True}} + docs = store.filter(filter_query) + assert len(docs) > 0 + for doc in docs: + assert doc.A + + filter_query = { + 'bool': { + 'filter': [ + {'terms': {'B': [3, 4, 7, 8]}}, + {'range': {'C': {'gte': 3, 'lte': 5}}}, + ] + } + } + docs = store.filter(filter_query) + assert [doc.id for doc in docs] == ['3', '4'] + + +def test_text_search(): + class MyDoc(BaseDoc): + text: str + + store = ElasticV7DocIndex[MyDoc]() + index_docs = [ + MyDoc(text='hello world'), + MyDoc(text='never gonna give you up'), + MyDoc(text='we are the world'), + ] + store.index(index_docs) + + query = 'world' + docs, scores = store.text_search(query, search_field='text') + + assert len(docs) == 2 + assert len(scores) == 2 + assert docs[0].text.index(query) >= 0 + assert docs[1].text.index(query) >= 0 + + queries = ['world', 'never'] + docs, scores = store.text_search_batched(queries, search_field='text') + for query, da, score in zip(queries, docs, scores): + assert len(da) > 0 + assert len(score) > 0 + for doc in da: + assert doc.text.index(query) >= 0 + + +def test_query_builder(): + class MyDoc(BaseDoc): + tens: NdArray[10] + num: int + text: str + + store = ElasticV7DocIndex[MyDoc]() + index_docs = [ + MyDoc( + id=f'{i}', tens=np.random.rand(10), num=int(i / 2), text=f'text {int(i/2)}' + ) + for i in range(10) + ] + store.index(index_docs) + + # build_query + q = store.build_query() + assert isinstance(q, store.QueryBuilder) + + # filter + q = store.build_query().filter({'term': {'num': 0}}).build() + docs, _ = store.execute_query(q) + assert [doc['id'] for doc in docs] == ['0', '1'] + + # find + q = store.build_query().find(index_docs[-1], search_field='tens', limit=3).build() + docs, scores = store.execute_query(q) + assert len(docs) == 3 + assert len(scores) == 3 + assert docs[0]['id'] == index_docs[-1].id + assert np.allclose(docs[0]['tens'], index_docs[-1].tens) + + # text search + q = store.build_query().text_search('0', search_field='text').build() + docs, _ = store.execute_query(q) + assert [doc['id'] for doc in docs] == ['0', '1'] + + # combination + q = ( + store.build_query() + .filter({'range': {'num': {'lte': 3}}}) + .find(index_docs[-1], search_field='tens') + .text_search('0', search_field='text') + .build() + ) + docs, _ = store.execute_query(q) + assert sorted([doc['id'] for doc in docs]) == ['0', '1'] + + # direct + index_docs = [ + MyDoc(id=f'{i}', tens=np.ones(10) * i, num=int(i / 2), text=f'text {int(i/2)}') + for i in range(10) + ] + store.index(index_docs) + + query = { + 'query': { + 'script_score': { + 'query': { + 'bool': { + 'filter': [ + {'range': {'num': {'gte': 2}}}, + {'range': {'num': {'lte': 3}}}, + ], + }, + }, + 'script': { + 'source': '1 / (1 + l2norm(params.query_vector, \'tens\'))', + 'params': {'query_vector': index_docs[-1].tens}, + }, + } + } + } + + docs, _ = store.execute_query(query) + assert [doc['id'] for doc in docs] == ['7', '6', '5', '4'] diff --git a/tests/index/elastic/v7/test_index_get_del.py b/tests/index/elastic/v7/test_index_get_del.py new file mode 100644 index 00000000000..8389e28ffbb --- /dev/null +++ b/tests/index/elastic/v7/test_index_get_del.py @@ -0,0 +1,280 @@ +from typing import Union + +import numpy as np +import pytest + +from docarray import BaseDoc, DocArray +from docarray.documents import ImageDoc, TextDoc +from docarray.index import ElasticV7DocIndex +from docarray.typing import NdArray +from tests.index.elastic.fixture import start_storage_v7 # noqa: F401 +from tests.index.elastic.fixture import DeepNestedDoc, FlatDoc, NestedDoc, SimpleDoc + + +@pytest.fixture +def ten_simple_docs(): + return [SimpleDoc(tens=np.random.randn(10)) for _ in range(10)] + + +@pytest.fixture +def ten_flat_docs(): + return [ + FlatDoc(tens_one=np.random.randn(10), tens_two=np.random.randn(50)) + for _ in range(10) + ] + + +@pytest.fixture +def ten_nested_docs(): + return [NestedDoc(d=SimpleDoc(tens=np.random.randn(10))) for _ in range(10)] + + +@pytest.fixture +def ten_deep_nested_docs(): + return [ + DeepNestedDoc(d=NestedDoc(d=SimpleDoc(tens=np.random.randn(10)))) + for _ in range(10) + ] + + +@pytest.mark.parametrize('use_docarray', [True, False]) +def test_index_simple_schema(ten_simple_docs, use_docarray): + store = ElasticV7DocIndex[SimpleDoc]() + if use_docarray: + ten_simple_docs = DocArray[SimpleDoc](ten_simple_docs) + + store.index(ten_simple_docs) + assert store.num_docs() == 10 + + +@pytest.mark.parametrize('use_docarray', [True, False]) +def test_index_flat_schema(ten_flat_docs, use_docarray): + store = ElasticV7DocIndex[FlatDoc]() + if use_docarray: + ten_flat_docs = DocArray[FlatDoc](ten_flat_docs) + + store.index(ten_flat_docs) + assert store.num_docs() == 10 + + +@pytest.mark.parametrize('use_docarray', [True, False]) +def test_index_nested_schema(ten_nested_docs, use_docarray): + store = ElasticV7DocIndex[NestedDoc]() + if use_docarray: + ten_nested_docs = DocArray[NestedDoc](ten_nested_docs) + + store.index(ten_nested_docs) + assert store.num_docs() == 10 + + +@pytest.mark.parametrize('use_docarray', [True, False]) +def test_index_deep_nested_schema(ten_deep_nested_docs, use_docarray): + store = ElasticV7DocIndex[DeepNestedDoc]() + if use_docarray: + ten_deep_nested_docs = DocArray[DeepNestedDoc](ten_deep_nested_docs) + + store.index(ten_deep_nested_docs) + assert store.num_docs() == 10 + + +def test_get_single(ten_simple_docs, ten_flat_docs, ten_nested_docs): + # simple + store = ElasticV7DocIndex[SimpleDoc]() + store.index(ten_simple_docs) + + assert store.num_docs() == 10 + for d in ten_simple_docs: + id_ = d.id + assert store[id_].id == id_ + assert np.all(store[id_].tens == d.tens) + + # flat + store = ElasticV7DocIndex[FlatDoc]() + store.index(ten_flat_docs) + + assert store.num_docs() == 10 + for d in ten_flat_docs: + id_ = d.id + assert store[id_].id == id_ + assert np.all(store[id_].tens_one == d.tens_one) + assert np.all(store[id_].tens_two == d.tens_two) + + # nested + store = ElasticV7DocIndex[NestedDoc]() + store.index(ten_nested_docs) + + assert store.num_docs() == 10 + for d in ten_nested_docs: + id_ = d.id + assert store[id_].id == id_ + assert store[id_].d.id == d.d.id + assert np.all(store[id_].d.tens == d.d.tens) + + +def test_get_multiple(ten_simple_docs, ten_flat_docs, ten_nested_docs): + docs_to_get_idx = [0, 2, 4, 6, 8] + + # simple + store = ElasticV7DocIndex[SimpleDoc]() + store.index(ten_simple_docs) + + assert store.num_docs() == 10 + docs_to_get = [ten_simple_docs[i] for i in docs_to_get_idx] + ids_to_get = [d.id for d in docs_to_get] + retrieved_docs = store[ids_to_get] + for id_, d_in, d_out in zip(ids_to_get, docs_to_get, retrieved_docs): + assert d_out.id == id_ + assert np.all(d_out.tens == d_in.tens) + + # flat + store = ElasticV7DocIndex[FlatDoc]() + store.index(ten_flat_docs) + + assert store.num_docs() == 10 + docs_to_get = [ten_flat_docs[i] for i in docs_to_get_idx] + ids_to_get = [d.id for d in docs_to_get] + retrieved_docs = store[ids_to_get] + for id_, d_in, d_out in zip(ids_to_get, docs_to_get, retrieved_docs): + assert d_out.id == id_ + assert np.all(d_out.tens_one == d_in.tens_one) + assert np.all(d_out.tens_two == d_in.tens_two) + + # nested + store = ElasticV7DocIndex[NestedDoc]() + store.index(ten_nested_docs) + + assert store.num_docs() == 10 + docs_to_get = [ten_nested_docs[i] for i in docs_to_get_idx] + ids_to_get = [d.id for d in docs_to_get] + retrieved_docs = store[ids_to_get] + for id_, d_in, d_out in zip(ids_to_get, docs_to_get, retrieved_docs): + assert d_out.id == id_ + assert d_out.d.id == d_in.d.id + assert np.all(d_out.d.tens == d_in.d.tens) + + +def test_get_key_error(ten_simple_docs): + store = ElasticV7DocIndex[SimpleDoc]() + store.index(ten_simple_docs) + + with pytest.raises(KeyError): + store['not_a_real_id'] + + +def test_persisting(ten_simple_docs): + store = ElasticV7DocIndex[SimpleDoc](index_name='test_persisting') + store.index(ten_simple_docs) + + store2 = ElasticV7DocIndex[SimpleDoc](index_name='test_persisting') + assert store2.num_docs() == 10 + + +def test_del_single(ten_simple_docs): + store = ElasticV7DocIndex[SimpleDoc]() + store.index(ten_simple_docs) + # delete once + assert store.num_docs() == 10 + del store[ten_simple_docs[0].id] + assert store.num_docs() == 9 + for i, d in enumerate(ten_simple_docs): + id_ = d.id + if i == 0: # deleted + with pytest.raises(KeyError): + store[id_] + else: + assert store[id_].id == id_ + assert np.all(store[id_].tens == d.tens) + # delete again + del store[ten_simple_docs[3].id] + assert store.num_docs() == 8 + for i, d in enumerate(ten_simple_docs): + id_ = d.id + if i in (0, 3): # deleted + with pytest.raises(KeyError): + store[id_] + else: + assert store[id_].id == id_ + assert np.all(store[id_].tens == d.tens) + + +def test_del_multiple(ten_simple_docs): + docs_to_del_idx = [0, 2, 4, 6, 8] + + store = ElasticV7DocIndex[SimpleDoc]() + store.index(ten_simple_docs) + + assert store.num_docs() == 10 + docs_to_del = [ten_simple_docs[i] for i in docs_to_del_idx] + ids_to_del = [d.id for d in docs_to_del] + del store[ids_to_del] + for i, doc in enumerate(ten_simple_docs): + if i in docs_to_del_idx: + with pytest.raises(KeyError): + store[doc.id] + else: + assert store[doc.id].id == doc.id + assert np.all(store[doc.id].tens == doc.tens) + + +def test_del_key_error(ten_simple_docs): + store = ElasticV7DocIndex[SimpleDoc]() + store.index(ten_simple_docs) + + with pytest.warns(UserWarning): + del store['not_a_real_id'] + + +def test_num_docs(ten_simple_docs): + store = ElasticV7DocIndex[SimpleDoc]() + store.index(ten_simple_docs) + + assert store.num_docs() == 10 + + del store[ten_simple_docs[0].id] + assert store.num_docs() == 9 + + del store[ten_simple_docs[3].id, ten_simple_docs[5].id] + assert store.num_docs() == 7 + + more_docs = [SimpleDoc(tens=np.random.rand(10)) for _ in range(5)] + store.index(more_docs) + assert store.num_docs() == 12 + + del store[more_docs[2].id, ten_simple_docs[7].id] + assert store.num_docs() == 10 + + +def test_index_union_doc(): + class MyDoc(BaseDoc): + tensor: Union[NdArray, str] + + class MySchema(BaseDoc): + tensor: NdArray + + store = ElasticV7DocIndex[MySchema]() + doc = [MyDoc(tensor=np.random.randn(128))] + store.index(doc) + + id_ = doc[0].id + assert store[id_].id == id_ + assert np.all(store[id_].tensor == doc[0].tensor) + + +def test_index_multi_modal_doc(): + class MyMultiModalDoc(BaseDoc): + image: ImageDoc + text: TextDoc + + store = ElasticV7DocIndex[MyMultiModalDoc]() + + doc = [ + MyMultiModalDoc( + image=ImageDoc(embedding=np.random.randn(128)), text=TextDoc(text='hello') + ) + ] + store.index(doc) + + id_ = doc[0].id + assert store[id_].id == id_ + assert np.all(store[id_].image.embedding == doc[0].image.embedding) + assert store[id_].text.text == doc[0].text.text From 2570be04ca36cf1ea0c4c09864ac71dd3e15ec77 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Wed, 29 Mar 2023 16:03:59 +0200 Subject: [PATCH 57/66] fix: add case for elastic search Signed-off-by: anna-charlotte --- docarray/index/__init__.py | 19 +++++++++++++------ docarray/utils/_internal/misc.py | 1 + 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/docarray/index/__init__.py b/docarray/index/__init__.py index 6b2ed9e9f8c..5856e4b9914 100644 --- a/docarray/index/__init__.py +++ b/docarray/index/__init__.py @@ -1,3 +1,4 @@ +import types from typing import TYPE_CHECKING from docarray.utils._internal.misc import ( @@ -12,15 +13,21 @@ def __getattr__(name: str): + lib: types.ModuleType if name == 'HnswDocumentIndex': import_library('hnswlib', raise_error=True) - from docarray.index.backends.hnswlib import HnswDocumentIndex # noqa - - if name not in __all__: - __all__.append(name) - - return HnswDocumentIndex + import docarray.index.backends.hnswlib as lib + elif name == 'ElasticV7DocIndex': + import_library('elasticsearch', raise_error=True) + import docarray.index.backends.elastic as lib else: raise ImportError( f'cannot import name \'{name}\' from \'{_get_path_from_docarray_root_level(__file__)}\'' ) + + index_cls = getattr(lib, name) + + if name not in __all__: + __all__.append(name) + + return index_cls diff --git a/docarray/utils/_internal/misc.py b/docarray/utils/_internal/misc.py index 362507d264d..e398b29997b 100644 --- a/docarray/utils/_internal/misc.py +++ b/docarray/utils/_internal/misc.py @@ -31,6 +31,7 @@ 'av': '"docarray[video]"', 'trimesh': '"docarray[mesh]"', 'hnswlib': '"docarray[hnswlib]"', + 'elasticsearch': '"docarray[elasticsearch]"', 'fastapi': '"docarray[web]"', 'torch': '"docarray[torch]"', 'tensorflow': 'protobuf==3.19.0 tensorflow', From 9fef7012e782e61a86b986790f8155627fa8fb95 Mon Sep 17 00:00:00 2001 From: Charlotte Gerhaher Date: Wed, 29 Mar 2023 16:06:29 +0200 Subject: [PATCH 58/66] refactor: map_docs_batch to map_docs_batched (#1312) Signed-off-by: anna-charlotte --- docarray/utils/map.py | 8 ++++---- docs/api_references/utils/maps_docs.md | 2 +- tests/benchmark_tests/test_map.py | 10 +++++----- tests/units/util/test_map.py | 6 +++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docarray/utils/map.py b/docarray/utils/map.py index 8ce4d974b18..31e93bc2175 100644 --- a/docarray/utils/map.py +++ b/docarray/utils/map.py @@ -1,4 +1,4 @@ -__all__ = ['map_docs', 'map_docs_batch'] +__all__ = ['map_docs', 'map_docs_batched'] from contextlib import nullcontext from math import ceil from multiprocessing.pool import Pool, ThreadPool @@ -103,7 +103,7 @@ def load_url_to_tensor(img: ImageDoc) -> ImageDoc: yield x -def map_docs_batch( +def map_docs_batched( da: T, func: Callable[[T], Union[T, T_doc]], batch_size: int, @@ -122,7 +122,7 @@ def map_docs_batch( ```python from docarray import BaseDoc, DocArray - from docarray.utils.map import map_docs_batch + from docarray.utils.map import map_docs_batched class MyDoc(BaseDoc): @@ -136,7 +136,7 @@ def upper_case_name(da: DocArray[MyDoc]) -> DocArray[MyDoc]: batch_size = 16 da = DocArray[MyDoc]([MyDoc(name='my orange cat') for _ in range(100)]) - it = map_docs_batch(da, upper_case_name, batch_size=batch_size) + it = map_docs_batched(da, upper_case_name, batch_size=batch_size) for i, d in enumerate(it): da[i * batch_size : (i + 1) * batch_size] = d diff --git a/docs/api_references/utils/maps_docs.md b/docs/api_references/utils/maps_docs.md index 9ccd971b1fd..da71bc867e9 100644 --- a/docs/api_references/utils/maps_docs.md +++ b/docs/api_references/utils/maps_docs.md @@ -1,7 +1,7 @@ # map ::: docarray.utils.map.map_docs -::: docarray.utils.map.map_docs_batch +::: docarray.utils.map.map_docs_batched diff --git a/tests/benchmark_tests/test_map.py b/tests/benchmark_tests/test_map.py index ace42dacfa7..d6018b9fdb0 100644 --- a/tests/benchmark_tests/test_map.py +++ b/tests/benchmark_tests/test_map.py @@ -7,7 +7,7 @@ from docarray import BaseDoc, DocArray from docarray.documents import ImageDoc from docarray.typing import NdArray -from docarray.utils.map import map_docs, map_docs_batch +from docarray.utils.map import map_docs, map_docs_batched from tests.units.typing.test_bytes import IMAGE_PATHS pytestmark = [pytest.mark.benchmark, pytest.mark.slow] @@ -56,7 +56,7 @@ def cpu_intensive_batch(da: DocArray[MyMatrix]) -> DocArray[MyMatrix]: return da -def test_map_docs_batch_multiprocessing(): +def test_map_docs_batched_multiprocessing(): if os.cpu_count() > 1: def time_multiprocessing(num_workers: int) -> float: @@ -66,7 +66,7 @@ def time_multiprocessing(num_workers: int) -> float: da = DocArray[MyMatrix]([MyMatrix(matrix=m) for m in matrices]) start_time = time() list( - map_docs_batch( + map_docs_batched( da=da, func=cpu_intensive_batch, batch_size=8, @@ -113,7 +113,7 @@ def io_intensive_batch(da: DocArray[ImageDoc]) -> DocArray[ImageDoc]: return da -def test_map_docs_batch_multithreading(): +def test_map_docs_batched_multithreading(): def time_multithreading_batch(num_workers: int) -> float: n_docs = 100 da = DocArray[ImageDoc]( @@ -121,7 +121,7 @@ def time_multithreading_batch(num_workers: int) -> float: ) start_time = time() list( - map_docs_batch( + map_docs_batched( da=da, func=io_intensive_batch, backend='thread', diff --git a/tests/units/util/test_map.py b/tests/units/util/test_map.py index 0e54aaa2732..c36ebc2f46e 100644 --- a/tests/units/util/test_map.py +++ b/tests/units/util/test_map.py @@ -5,7 +5,7 @@ from docarray import BaseDoc, DocArray from docarray.documents import ImageDoc from docarray.typing import ImageUrl, NdArray -from docarray.utils.map import map_docs, map_docs_batch +from docarray.utils.map import map_docs, map_docs_batched from tests.units.typing.test_bytes import IMAGE_PATHS N_DOCS = 2 @@ -73,10 +73,10 @@ class MyImage(BaseDoc): @pytest.mark.slow @pytest.mark.parametrize('n_docs,batch_size', [(10, 5), (10, 8)]) @pytest.mark.parametrize('backend', ['thread', 'process']) -def test_map_docs_batch(n_docs, batch_size, backend): +def test_map_docs_batched(n_docs, batch_size, backend): da = DocArray[MyImage]([MyImage(url=IMAGE_PATHS['png']) for _ in range(n_docs)]) - it = map_docs_batch( + it = map_docs_batched( da=da, func=load_from_da, batch_size=batch_size, backend=backend ) assert isinstance(it, Generator) From a0264f04baa218c1fe2637af23c0b17822d86ba6 Mon Sep 17 00:00:00 2001 From: Charlotte Gerhaher Date: Wed, 29 Mar 2023 16:07:17 +0200 Subject: [PATCH 59/66] refactor: map_docs_batch to map_docs_batched (#1312) Signed-off-by: anna-charlotte From fefbe86f716996d7f74bff79ff0fd300a1aecaab Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Wed, 29 Mar 2023 16:59:39 +0200 Subject: [PATCH 60/66] fix: clean up Signed-off-by: anna-charlotte --- docarray/index/__init__.py | 1 + docarray/store/__init__.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docarray/index/__init__.py b/docarray/index/__init__.py index 5856e4b9914..37ef5e6f611 100644 --- a/docarray/index/__init__.py +++ b/docarray/index/__init__.py @@ -7,6 +7,7 @@ ) if TYPE_CHECKING: + from docarray.index.backends.elastic import ElasticV7DocIndex # noqa: F401 from docarray.index.backends.hnswlib import HnswDocumentIndex # noqa: F401 __all__ = [] diff --git a/docarray/store/__init__.py b/docarray/store/__init__.py index e21892bcdd6..9547db27c3e 100644 --- a/docarray/store/__init__.py +++ b/docarray/store/__init__.py @@ -29,9 +29,9 @@ def __getattr__(name: str): f'cannot import name \'{name}\' from \'{_get_path_from_docarray_root_level(__file__)}\'' ) - tensor_cls = getattr(lib, name) + store_cls = getattr(lib, name) if name not in __all__: __all__.append(name) - return tensor_cls + return store_cls From 2557a6119d96ca9d74369f62e366dcc4011be92c Mon Sep 17 00:00:00 2001 From: Aman Agarwal Date: Wed, 29 Mar 2023 22:18:51 +0530 Subject: [PATCH 61/66] feat: torch backend basic operation tests (#1306) Signed-off-by: agaraman0 Signed-off-by: anna-charlotte --- .../torch_backend/test_basics.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/units/computation_backends/torch_backend/test_basics.py b/tests/units/computation_backends/torch_backend/test_basics.py index eab546615f0..925846e2b0c 100644 --- a/tests/units/computation_backends/torch_backend/test_basics.py +++ b/tests/units/computation_backends/torch_backend/test_basics.py @@ -1,3 +1,4 @@ +import numpy as np import pytest import torch @@ -100,3 +101,46 @@ def test_minmax_normalize(array, t_range, x_range, result): tensor=array, t_range=t_range, x_range=x_range ) assert torch.allclose(output, result) + + +def test_reshape(): + a = torch.tensor([[[1, 2, 3], [4, 5, 6]]]) + b = TorchCompBackend.reshape(a, (2, 3)) + assert torch.equal(b, torch.tensor([[1, 2, 3], [4, 5, 6]])) + + +def test_copy(): + a = torch.tensor([1, 2, 3]) + b = TorchCompBackend.copy(a) + assert torch.equal(a, b) + + +def test_stack(): + a = torch.tensor([1, 2, 3]) + b = torch.tensor([4, 5, 6]) + stacked = TorchCompBackend.stack([a, b], dim=0) + assert torch.equal(stacked, torch.tensor([[1, 2, 3], [4, 5, 6]])) + + +def test_empty_all(): + shape = (2, 3) + dtype = torch.float32 + device = 'cpu' + a = TorchCompBackend.empty(shape, dtype, device) + assert a.shape == shape and a.dtype == dtype and a.device.type == device + + +def test_to_numpy(): + a = torch.tensor([1, 2, 3]) + b = TorchCompBackend.to_numpy(a) + assert np.array_equal(b, np.array(a)) + + +def test_none_value(): + assert torch.isnan(TorchCompBackend.none_value()) + + +def test_detach(): + a = torch.tensor([1.0, 2.0, 3.0], requires_grad=True) + b = TorchCompBackend.detach(a) + assert not b.requires_grad From 3509d2735b5b921a275dd7b3f60d82a57e0be72a Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Thu, 30 Mar 2023 09:06:31 +0200 Subject: [PATCH 62/66] fix: ci add --fix-missing to apt-get Signed-off-by: anna-charlotte --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d289509f5d4..24f59602e7c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,7 +112,7 @@ jobs: python -m pip install --upgrade pip python -m pip install poetry poetry install --all-extras - sudo apt-get install --no-install-recommends ffmpeg + sudo apt-get install --no-install-recommends ffmpeg --fix-missing - name: Test id: test @@ -157,7 +157,7 @@ jobs: python -m pip install poetry rm poetry.lock poetry install --all-extras - sudo apt-get install --no-install-recommends ffmpeg + sudo apt-get install --no-install-recommends ffmpeg --fix-missing - name: Test id: test @@ -187,7 +187,7 @@ jobs: python -m pip install poetry poetry install --all-extras poetry run pip install protobuf==3.19.0 # we check that we support 3.19 - sudo apt-get install --no-install-recommends ffmpeg + sudo apt-get install --no-install-recommends ffmpeg --fix-missing - name: Test id: test @@ -214,7 +214,7 @@ jobs: python -m pip install --upgrade pip python -m pip install poetry poetry install --all-extras - sudo apt-get install --no-install-recommends ffmpeg + sudo apt-get install --no-install-recommends ffmpeg --fix-missing - name: Test id: test @@ -242,7 +242,7 @@ jobs: poetry install --all-extras poetry run pip install protobuf==3.19.0 poetry run pip install tensorflow==2.11.0 - sudo apt-get install --no-install-recommends ffmpeg + sudo apt-get install --no-install-recommends ffmpeg --fix-missing - name: Test id: test From 81de695b5644655f3cd5a34f9542b5d1af63c65f Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Thu, 30 Mar 2023 09:11:50 +0200 Subject: [PATCH 63/66] fix: revert "fix: ci add --fix-missing to apt-get" Signed-off-by: anna-charlotte --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24f59602e7c..d289509f5d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,7 +112,7 @@ jobs: python -m pip install --upgrade pip python -m pip install poetry poetry install --all-extras - sudo apt-get install --no-install-recommends ffmpeg --fix-missing + sudo apt-get install --no-install-recommends ffmpeg - name: Test id: test @@ -157,7 +157,7 @@ jobs: python -m pip install poetry rm poetry.lock poetry install --all-extras - sudo apt-get install --no-install-recommends ffmpeg --fix-missing + sudo apt-get install --no-install-recommends ffmpeg - name: Test id: test @@ -187,7 +187,7 @@ jobs: python -m pip install poetry poetry install --all-extras poetry run pip install protobuf==3.19.0 # we check that we support 3.19 - sudo apt-get install --no-install-recommends ffmpeg --fix-missing + sudo apt-get install --no-install-recommends ffmpeg - name: Test id: test @@ -214,7 +214,7 @@ jobs: python -m pip install --upgrade pip python -m pip install poetry poetry install --all-extras - sudo apt-get install --no-install-recommends ffmpeg --fix-missing + sudo apt-get install --no-install-recommends ffmpeg - name: Test id: test @@ -242,7 +242,7 @@ jobs: poetry install --all-extras poetry run pip install protobuf==3.19.0 poetry run pip install tensorflow==2.11.0 - sudo apt-get install --no-install-recommends ffmpeg --fix-missing + sudo apt-get install --no-install-recommends ffmpeg - name: Test id: test From b41846208f421dd4777155786cf53169ace0885e Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Thu, 30 Mar 2023 09:14:07 +0200 Subject: [PATCH 64/66] fix: ci apt-get update Signed-off-by: anna-charlotte --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d289509f5d4..722e2525322 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,6 +112,7 @@ jobs: python -m pip install --upgrade pip python -m pip install poetry poetry install --all-extras + sudo apt-get update sudo apt-get install --no-install-recommends ffmpeg - name: Test @@ -157,6 +158,7 @@ jobs: python -m pip install poetry rm poetry.lock poetry install --all-extras + sudo apt-get update sudo apt-get install --no-install-recommends ffmpeg - name: Test @@ -187,6 +189,7 @@ jobs: python -m pip install poetry poetry install --all-extras poetry run pip install protobuf==3.19.0 # we check that we support 3.19 + sudo apt-get update sudo apt-get install --no-install-recommends ffmpeg - name: Test @@ -214,6 +217,7 @@ jobs: python -m pip install --upgrade pip python -m pip install poetry poetry install --all-extras + sudo apt-get update sudo apt-get install --no-install-recommends ffmpeg - name: Test @@ -242,6 +246,7 @@ jobs: poetry install --all-extras poetry run pip install protobuf==3.19.0 poetry run pip install tensorflow==2.11.0 + sudo apt-get update sudo apt-get install --no-install-recommends ffmpeg - name: Test From fb01b6b9845f59c121de556f22d1d459ec49cd71 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Thu, 30 Mar 2023 11:36:42 +0200 Subject: [PATCH 65/66] fix: apply samis suggestions from code review Signed-off-by: anna-charlotte --- docarray/proto/pb/docarray_pb2.py | 1 - tests/documentation/test_docstring.py | 3 --- 2 files changed, 4 deletions(-) diff --git a/docarray/proto/pb/docarray_pb2.py b/docarray/proto/pb/docarray_pb2.py index f6eeaf8fdb9..a830f17ddc4 100644 --- a/docarray/proto/pb/docarray_pb2.py +++ b/docarray/proto/pb/docarray_pb2.py @@ -2,7 +2,6 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: docarray.proto """Generated protocol buffer code.""" - from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool diff --git a/tests/documentation/test_docstring.py b/tests/documentation/test_docstring.py index 4848601ff72..91bc43468c1 100644 --- a/tests/documentation/test_docstring.py +++ b/tests/documentation/test_docstring.py @@ -47,9 +47,6 @@ def get_obj_to_check(lib): for obj in obj_to_check: members.extend(get_codeblock_members(obj)) -print(f"len(members) = {len(members)}") -print(f"len(set(members)) = {len(set(members))}") - @pytest.mark.parametrize("obj", members, ids=lambda d: d.__qualname__) def test_member(obj): From a40c44f81bb290c1f11da69e25eafa7bc5717b12 Mon Sep 17 00:00:00 2001 From: anna-charlotte Date: Thu, 30 Mar 2023 11:50:42 +0200 Subject: [PATCH 66/66] fix: apply samis suggestions from code review Signed-off-by: anna-charlotte --- docarray/utils/_internal/misc.py | 2 +- tests/units/typing/tensor/test_cross_backend.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docarray/utils/_internal/misc.py b/docarray/utils/_internal/misc.py index e398b29997b..9f97ad88ea6 100644 --- a/docarray/utils/_internal/misc.py +++ b/docarray/utils/_internal/misc.py @@ -52,7 +52,7 @@ def import_library( lib = None if lib is None and raise_error: - raise RuntimeError( + raise ImportError( f'The following required library is not installed: {package} \n' f'To install all necessary libraries, run: `pip install {INSTALL_INSTRUCTIONS[package]}`.' ) diff --git a/tests/units/typing/tensor/test_cross_backend.py b/tests/units/typing/tensor/test_cross_backend.py index 4e4528c0502..702cd678d6f 100644 --- a/tests/units/typing/tensor/test_cross_backend.py +++ b/tests/units/typing/tensor/test_cross_backend.py @@ -6,7 +6,7 @@ try: from docarray.typing import TensorFlowTensor -except RuntimeError: +except (ImportError, TypeError): pass