docarray · samsja · Nov 21, 2022 · Nov 11, 2022 · Nov 11, 2022 · Nov 15, 2022
diff --git a/docarray/document/mixins/proto.py b/docarray/document/mixins/proto.py
@@ -3,7 +3,15 @@
 from docarray.document.abstract_document import AbstractDocument
 from docarray.document.base_node import BaseNode
 from docarray.proto import DocumentProto, NodeProto
-from docarray.typing import ID, AnyUrl, Embedding, ImageUrl, Tensor, TorchTensor
+from docarray.typing import (
+    ID,
+    AnyUrl,
+    Embedding,
+    ImageUrl,
+    Tensor,
+    TextUrl,
+    TorchTensor,
+)
 
 T = TypeVar('T', bound='ProtoMixin')
 
@@ -27,6 +35,7 @@ def from_protobuf(cls: Type[T], pb_msg: 'DocumentProto') -> T:
                 torch_tensor=TorchTensor,
                 embedding=Embedding,
                 any_url=AnyUrl,
+                text_url=TextUrl,
                 image_url=ImageUrl,
                 id=ID,
             )

diff --git a/docarray/proto/docarray.proto b/docarray/proto/docarray.proto
@@ -53,9 +53,11 @@ message NodeProto {
 
     string image_url = 8;
 
-    string id = 9;
+    string text_url = 9;
 
-    NdArrayProto torch_tensor = 10;
+    string id = 10;
+
+    NdArrayProto torch_tensor = 11;
 
   }
 

diff --git a/docarray/proto/pb2/docarray_pb2.py b/docarray/proto/pb2/docarray_pb2.py
diff --git a/docarray/typing/__init__.py b/docarray/typing/__init__.py
@@ -1,13 +1,14 @@
 from docarray.typing.id import ID
 from docarray.typing.tensor import Tensor, TorchTensor
 from docarray.typing.tensor.embedding import Embedding
-from docarray.typing.url import AnyUrl, ImageUrl
+from docarray.typing.url import AnyUrl, ImageUrl, TextUrl
 
 __all__ = [
     'TorchTensor',
     'Tensor',
     'Embedding',
     'ImageUrl',
+    'TextUrl',
     'AnyUrl',
     'ID',
 ]
diff --git a/docarray/typing/url/__init__.py b/docarray/typing/url/__init__.py
@@ -1,4 +1,5 @@
 from docarray.typing.url.any_url import AnyUrl
 from docarray.typing.url.image_url import ImageUrl
+from docarray.typing.url.text_url import TextUrl
 
-__all__ = ['ImageUrl', 'AnyUrl']
+__all__ = ['ImageUrl', 'AnyUrl', 'TextUrl']
diff --git a/docarray/typing/url/text_url.py b/docarray/typing/url/text_url.py
@@ -0,0 +1,91 @@
+from typing import Optional
+
+from docarray.proto import NodeProto
+from docarray.typing.url.any_url import AnyUrl
+from docarray.typing.url.helper import _uri_to_blob
+
+
+class TextUrl(AnyUrl):
+    """
+    URL to a text file.
+    Cane be remote (web) URL, or a local file path.
+    """
+
+    def _to_node_protobuf(self) -> NodeProto:
+        """Convert Document into a NodeProto protobuf message. This function should
+        be called when the Document is nested into another Document that need to
+        be converted into a protobuf
+
+        :return: the nested item protobuf message
+        """
+        return NodeProto(text_url=str(self))
+
+    def load_to_bytes(self, timeout: Optional[float] = None) -> bytes:
+        """
+        Load the text file into a bytes object.
+
+        EXAMPLE USAGE
+
+        .. code-block:: python
+
+            from docarray import Document
+            from docarray.typing import TextUrl
+
+
+            class MyDoc(Document):
+                remote_url: TextUrl
+                local_url: TextUrl
+
+
+            doc = MyDoc(
+                remote_url='https://de.wikipedia.org/wiki/Brixen',
+                local_url='home/username/my_file.txt',
+            )
+
+            remote_txt_bytes = doc.remote_url.load_to_bytes()
+            local_txt_bytes = doc.local_url.load_to_bytes()
+
+        :param timeout: timeout (sec) for urlopen network request.
+            Only relevant if URL is not local
+        :return: the text file content as bytes
+        """
+        return _uri_to_blob(self, timeout=timeout)
+
+    def load(self, charset: str = 'utf-8', timeout: Optional[float] = None) -> str:
+        """
+        Load the text file into a string.
+
+        EXAMPLE USAGE
+
+        .. code-block:: python
+
+            from docarray import Document
+            from docarray.typing import TextUrl
+
+
+            class MyDoc(Document):
+                remote_url: TextUrl
+                local_url: TextUrl
+
+
+            doc = MyDoc(
+                remote_url='https://de.wikipedia.org/wiki/Brixen',
+                local_url='home/username/my_file.txt',
+            )
+
+            remote_txt = doc.remote_url.load()
+            print(remote_txt)
+            # prints: ```<!DOCTYPE html>\n<html class="client-nojs" ... > ...```
+
+            local_txt = doc.local_url.load()
+            print(local_txt)
+            # prints content of my_file.txt
+
+
+        :param timeout: timeout (sec) for urlopen network request.
+            Only relevant if URL is not local
+        :param charset: decoding charset; may be any character set registered with IANA
+        :return: the text file content
+        """
+        _bytes = _uri_to_blob(self, timeout=timeout)
+        return _bytes.decode(charset)
diff --git a/tests/integrations/typing/test_typing_proto.py b/tests/integrations/typing/test_typing_proto.py
@@ -3,7 +3,7 @@
 
 from docarray import Document
 from docarray.document import AnyDocument
-from docarray.typing import AnyUrl, Embedding, ImageUrl, Tensor, TorchTensor
+from docarray.typing import AnyUrl, Embedding, ImageUrl, Tensor, TextUrl, TorchTensor
 
 
 def test_proto_all_types():
@@ -13,13 +13,15 @@ class Mymmdoc(Document):
         embedding: Embedding
         any_url: AnyUrl
         image_url: ImageUrl
+        text_url: TextUrl
 
     doc = Mymmdoc(
         tensor=np.zeros((3, 224, 224)),
         torch_tensor=torch.zeros((3, 224, 224)),
         embedding=np.zeros((100, 1)),
         any_url='http://jina.ai',
         image_url='http://jina.ai/bla.jpg',
+        text_url='http://jina.ai',
     )
 
     new_doc = AnyDocument.from_protobuf(doc.to_protobuf())