From 372b4880fbca786552d7368d1de7c51efcddfc6b Mon Sep 17 00:00:00 2001 From: Silas Marvin <19626586+SilasMarvin@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:09:06 -0700 Subject: [PATCH 1/3] Bump to work with python3.12 --- pgml-sdks/pgml/python/manual-build-deploy.sh | 2 +- pgml-sdks/pgml/python/pgml/pgml.pyi | 121 +++++++++++++++++++ 2 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 pgml-sdks/pgml/python/pgml/pgml.pyi diff --git a/pgml-sdks/pgml/python/manual-build-deploy.sh b/pgml-sdks/pgml/python/manual-build-deploy.sh index d4743610d..8e77ca385 100755 --- a/pgml-sdks/pgml/python/manual-build-deploy.sh +++ b/pgml-sdks/pgml/python/manual-build-deploy.sh @@ -3,4 +3,4 @@ echo "Make sure and set the environment variable MATURIN_PYPI_TOKEN to your PyPI token." cd .. -PYTHON_STUB_FILE="python/pgml/pgml.pyi" maturin publish -r $1 -i python3.8 -i python3.9 -i python3.10 -i python3.11 --skip-existing -F python +PYTHON_STUB_FILE="python/pgml/pgml.pyi" maturin publish -r $1 -i python3.8 -i python3.9 -i python3.10 -i python3.11 -i python3.12 --skip-existing -F python diff --git a/pgml-sdks/pgml/python/pgml/pgml.pyi b/pgml-sdks/pgml/python/pgml/pgml.pyi new file mode 100644 index 000000000..111f9863c --- /dev/null +++ b/pgml-sdks/pgml/python/pgml/pgml.pyi @@ -0,0 +1,121 @@ + +def init_logger(level: Optional[str] = "", format: Optional[str] = "") -> None +def SingleFieldPipeline(name: str, model: Optional[Model] = None, splitter: Optional[Splitter] = None, parameters: Optional[Json] = Any) -> Pipeline +async def migrate() -> None + +Json = Any +DateTime = int +GeneralJsonIterator = Any +GeneralJsonAsyncIterator = Any + +# Top of file key: A12BECOD! +from typing import List, Dict, Optional, Self, Any + + +class Builtins: + def __init__(self, database_url: Optional[str] = "Default set in Rust. Please check the documentation.") -> Self + ... + def query(self, query: str) -> QueryRunner + ... + async def transform(self, task: Json, inputs: List[str], args: Optional[Json] = Any) -> Json + ... + +class Collection: + def __init__(self, name: str, database_url: Optional[str] = "Default set in Rust. Please check the documentation.") -> Self + ... + async def add_pipeline(self, pipeline: Pipeline) -> None + ... + async def remove_pipeline(self, pipeline: Pipeline) -> None + ... + async def enable_pipeline(self, pipeline: Pipeline) -> None + ... + async def disable_pipeline(self, pipeline: Pipeline) -> None + ... + async def upsert_documents(self, documents: List[Json], args: Optional[Json] = Any) -> None + ... + async def get_documents(self, args: Optional[Json] = Any) -> List[Json] + ... + async def delete_documents(self, filter: Json) -> None + ... + async def search(self, query: Json, pipeline: Pipeline) -> Json + ... + async def add_search_event(self, search_id: int, search_result: int, event: Json, pipeline: Pipeline) -> None + ... + async def vector_search(self, query: Json, pipeline: Pipeline) -> List[Json] + ... + async def archive(self) -> None + ... + def query(self) -> QueryBuilder + ... + async def get_pipelines(self) -> List[Pipeline] + ... + async def get_pipeline(self, name: str) -> Pipeline + ... + async def exists(self) -> bool + ... + async def upsert_directory(self, path: str, args: Json) -> None + ... + async def get_pipeline_status(self, pipeline: Pipeline) -> Json + ... + async def generate_er_diagram(self, pipeline: Pipeline) -> str + ... + async def upsert_file(self, path: str) -> None + ... + +class Model: + def __init__(self, name: Optional[str] = "Default set in Rust. Please check the documentation.", source: Optional[str] = "Default set in Rust. Please check the documentation.", parameters: Optional[Json] = Any) -> Self + ... + +class OpenSourceAI: + def __init__(self, database_url: Optional[str] = "Default set in Rust. Please check the documentation.") -> Self + ... + async def chat_completions_create_stream_async(self, model: Json, messages: List[Json], max_tokens: Optional[Any] = Any, temperature: Optional[float] = 1, n: Optional[Any] = Any, chat_template: Optional[str] = "Default set in Rust. Please check the documentation.") -> GeneralJsonAsyncIterator + ... + def chat_completions_create_stream(self, model: Json, messages: List[Json], max_tokens: Optional[Any] = Any, temperature: Optional[float] = 1, n: Optional[Any] = Any, chat_template: Optional[str] = "Default set in Rust. Please check the documentation.") -> GeneralJsonIterator + ... + async def chat_completions_create_async(self, model: Json, messages: List[Json], max_tokens: Optional[Any] = Any, temperature: Optional[float] = 1, n: Optional[Any] = Any, chat_template: Optional[str] = "Default set in Rust. Please check the documentation.") -> Json + ... + def chat_completions_create(self, model: Json, messages: List[Json], max_tokens: Optional[Any] = Any, temperature: Optional[float] = 1, n: Optional[Any] = Any, chat_template: Optional[str] = "Default set in Rust. Please check the documentation.") -> Json + ... + +class Pipeline: + def __init__(self, name: str, schema: Optional[Json] = Any) -> Self + ... + +class QueryBuilder: + def limit(self, limit: int) -> Self + ... + def filter(self, filter: Json) -> Self + ... + def vector_recall(self, query: str, pipeline: Pipeline, query_parameters: Optional[Json] = Any) -> Self + ... + async def fetch_all(self) -> List[tuple[float, str, Json]] + ... + +class QueryRunner: + async def fetch_all(self) -> Json + ... + async def execute(self) -> None + ... + def bind_string(self, bind_value: str) -> Self + ... + def bind_int(self, bind_value: int) -> Self + ... + def bind_float(self, bind_value: float) -> Self + ... + def bind_bool(self, bind_value: bool) -> Self + ... + def bind_json(self, bind_value: Json) -> Self + ... + +class Splitter: + def __init__(self, name: Optional[str] = "Default set in Rust. Please check the documentation.", parameters: Optional[Json] = Any) -> Self + ... + +class TransformerPipeline: + def __init__(self, task: str, model: Optional[str] = "Default set in Rust. Please check the documentation.", args: Optional[Json] = Any, database_url: Optional[str] = "Default set in Rust. Please check the documentation.") -> Self + ... + async def transform(self, inputs: List[Json], args: Optional[Json] = Any) -> Json + ... + async def transform_stream(self, input: Json, args: Optional[Json] = Any, batch_size: Optional[Any] = Any) -> GeneralJsonAsyncIterator + ... From da1a9a30a9e41ceb0fe65aa8f9b2ecfc459ed835 Mon Sep 17 00:00:00 2001 From: Silas Marvin <19626586+SilasMarvin@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:12:03 -0700 Subject: [PATCH 2/3] Remove unnecessary file --- pgml-sdks/pgml/python/pgml/pgml.pyi | 121 ---------------------------- 1 file changed, 121 deletions(-) delete mode 100644 pgml-sdks/pgml/python/pgml/pgml.pyi diff --git a/pgml-sdks/pgml/python/pgml/pgml.pyi b/pgml-sdks/pgml/python/pgml/pgml.pyi deleted file mode 100644 index 111f9863c..000000000 --- a/pgml-sdks/pgml/python/pgml/pgml.pyi +++ /dev/null @@ -1,121 +0,0 @@ - -def init_logger(level: Optional[str] = "", format: Optional[str] = "") -> None -def SingleFieldPipeline(name: str, model: Optional[Model] = None, splitter: Optional[Splitter] = None, parameters: Optional[Json] = Any) -> Pipeline -async def migrate() -> None - -Json = Any -DateTime = int -GeneralJsonIterator = Any -GeneralJsonAsyncIterator = Any - -# Top of file key: A12BECOD! -from typing import List, Dict, Optional, Self, Any - - -class Builtins: - def __init__(self, database_url: Optional[str] = "Default set in Rust. Please check the documentation.") -> Self - ... - def query(self, query: str) -> QueryRunner - ... - async def transform(self, task: Json, inputs: List[str], args: Optional[Json] = Any) -> Json - ... - -class Collection: - def __init__(self, name: str, database_url: Optional[str] = "Default set in Rust. Please check the documentation.") -> Self - ... - async def add_pipeline(self, pipeline: Pipeline) -> None - ... - async def remove_pipeline(self, pipeline: Pipeline) -> None - ... - async def enable_pipeline(self, pipeline: Pipeline) -> None - ... - async def disable_pipeline(self, pipeline: Pipeline) -> None - ... - async def upsert_documents(self, documents: List[Json], args: Optional[Json] = Any) -> None - ... - async def get_documents(self, args: Optional[Json] = Any) -> List[Json] - ... - async def delete_documents(self, filter: Json) -> None - ... - async def search(self, query: Json, pipeline: Pipeline) -> Json - ... - async def add_search_event(self, search_id: int, search_result: int, event: Json, pipeline: Pipeline) -> None - ... - async def vector_search(self, query: Json, pipeline: Pipeline) -> List[Json] - ... - async def archive(self) -> None - ... - def query(self) -> QueryBuilder - ... - async def get_pipelines(self) -> List[Pipeline] - ... - async def get_pipeline(self, name: str) -> Pipeline - ... - async def exists(self) -> bool - ... - async def upsert_directory(self, path: str, args: Json) -> None - ... - async def get_pipeline_status(self, pipeline: Pipeline) -> Json - ... - async def generate_er_diagram(self, pipeline: Pipeline) -> str - ... - async def upsert_file(self, path: str) -> None - ... - -class Model: - def __init__(self, name: Optional[str] = "Default set in Rust. Please check the documentation.", source: Optional[str] = "Default set in Rust. Please check the documentation.", parameters: Optional[Json] = Any) -> Self - ... - -class OpenSourceAI: - def __init__(self, database_url: Optional[str] = "Default set in Rust. Please check the documentation.") -> Self - ... - async def chat_completions_create_stream_async(self, model: Json, messages: List[Json], max_tokens: Optional[Any] = Any, temperature: Optional[float] = 1, n: Optional[Any] = Any, chat_template: Optional[str] = "Default set in Rust. Please check the documentation.") -> GeneralJsonAsyncIterator - ... - def chat_completions_create_stream(self, model: Json, messages: List[Json], max_tokens: Optional[Any] = Any, temperature: Optional[float] = 1, n: Optional[Any] = Any, chat_template: Optional[str] = "Default set in Rust. Please check the documentation.") -> GeneralJsonIterator - ... - async def chat_completions_create_async(self, model: Json, messages: List[Json], max_tokens: Optional[Any] = Any, temperature: Optional[float] = 1, n: Optional[Any] = Any, chat_template: Optional[str] = "Default set in Rust. Please check the documentation.") -> Json - ... - def chat_completions_create(self, model: Json, messages: List[Json], max_tokens: Optional[Any] = Any, temperature: Optional[float] = 1, n: Optional[Any] = Any, chat_template: Optional[str] = "Default set in Rust. Please check the documentation.") -> Json - ... - -class Pipeline: - def __init__(self, name: str, schema: Optional[Json] = Any) -> Self - ... - -class QueryBuilder: - def limit(self, limit: int) -> Self - ... - def filter(self, filter: Json) -> Self - ... - def vector_recall(self, query: str, pipeline: Pipeline, query_parameters: Optional[Json] = Any) -> Self - ... - async def fetch_all(self) -> List[tuple[float, str, Json]] - ... - -class QueryRunner: - async def fetch_all(self) -> Json - ... - async def execute(self) -> None - ... - def bind_string(self, bind_value: str) -> Self - ... - def bind_int(self, bind_value: int) -> Self - ... - def bind_float(self, bind_value: float) -> Self - ... - def bind_bool(self, bind_value: bool) -> Self - ... - def bind_json(self, bind_value: Json) -> Self - ... - -class Splitter: - def __init__(self, name: Optional[str] = "Default set in Rust. Please check the documentation.", parameters: Optional[Json] = Any) -> Self - ... - -class TransformerPipeline: - def __init__(self, task: str, model: Optional[str] = "Default set in Rust. Please check the documentation.", args: Optional[Json] = Any, database_url: Optional[str] = "Default set in Rust. Please check the documentation.") -> Self - ... - async def transform(self, inputs: List[Json], args: Optional[Json] = Any) -> Json - ... - async def transform_stream(self, input: Json, args: Optional[Json] = Any, batch_size: Optional[Any] = Any) -> GeneralJsonAsyncIterator - ... From 04847bb3a2d75fa83b01185e0b06689677797a35 Mon Sep 17 00:00:00 2001 From: Silas Marvin <19626586+SilasMarvin@users.noreply.github.com> Date: Mon, 15 Apr 2024 14:12:42 -0700 Subject: [PATCH 3/3] Bump to work with python3.12 --- .github/workflows/python-sdk.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/python-sdk.yml b/.github/workflows/python-sdk.yml index e8d042fff..4ad7dec5b 100644 --- a/.github/workflows/python-sdk.yml +++ b/.github/workflows/python-sdk.yml @@ -41,6 +41,7 @@ jobs: python3.9 python3.9-dev \ python3.10 python3.10-dev \ python3.11 python3.11-dev \ + python3.12 python3.12-dev \ python3-pip \ git pip install maturin @@ -50,13 +51,13 @@ jobs: env: MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} PYTHON_STUB_FILE: "python/pgml/pgml.pyi" - run: maturin publish -r testpypi -i python3.7 -i python3.8 -i python3.9 -i python3.10 -i python3.11 --skip-existing -F python + run: maturin publish -r testpypi -i python3.7 -i python3.8 -i python3.9 -i python3.10 -i python3.11 -i python3.12 --skip-existing -F python - name: Build and deploy wheels to PyPI if: github.event.inputs.deploy_to_pypi == 'true' env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} PYTHON_STUB_FILE: "python/pgml/pgml.pyi" - run: maturin publish -i python3.7 -i python3.8 -i python3.9 -i python3.10 -i python3.11 --skip-existing -F python + run: maturin publish -i python3.7 -i python3.8 -i python3.9 -i python3.10 -i python3.11 -i python3.12 --skip-existing -F python deploy-python-sdk-mac: runs-on: macos-latest @@ -80,25 +81,26 @@ jobs: brew install python@3.9 brew install python@3.10 brew install python@3.11 + brew install python@3.12 pip3 install maturin - name: Build and deploy wheels to TestPyPI if: github.event.inputs.deploy_to_pypi == 'false' env: MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} PYTHON_STUB_FILE: "python/pgml/pgml.pyi" - run: maturin publish -r testpypi -i python3.8 -i python3.9 -i python3.10 -i python3.11 --skip-existing -F python + run: maturin publish -r testpypi -i python3.8 -i python3.9 -i python3.10 -i python3.11 -i python3.12 --skip-existing -F python - name: Build and deploy wheels to PyPI if: github.event.inputs.deploy_to_pypi == 'true' env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} PYTHON_STUB_FILE: "python/pgml/pgml.pyi" - run: maturin publish -i python3.8 -i python3.9 -i python3.10 -i python3.11 --skip-existing -F python + run: maturin publish -i python3.8 -i python3.9 -i python3.10 -i python3.11 -i python3.12 --skip-existing -F python deploy-python-sdk-windows: runs-on: windows-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] defaults: run: working-directory: pgml-sdks\pgml @@ -124,10 +126,10 @@ jobs: env: MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} PYTHON_STUB_FILE: "python/pgml/pgml.pyi" - run: maturin publish -r testpypi -i python3.8 -i python3.9 -i python3.10 -i python3.11 --skip-existing -F python + run: maturin publish -r testpypi -i python3.8 -i python3.9 -i python3.10 -i python3.11 -i python3.12 --skip-existing -F python - name: Build and deploy wheels to PyPI if: github.event.inputs.deploy_to_pypi == 'true' env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} PYTHON_STUB_FILE: "python/pgml/pgml.pyi" - run: maturin publish -i python3.8 -i python3.9 -i python3.10 -i python3.11 --skip-existing -F python + run: maturin publish -i python3.8 -i python3.9 -i python3.10 -i python3.11 -i python3.12 --skip-existing -F python