From 5511c2fc1cb7689532627a3b704f4695ba4dce06 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Tue, 17 Feb 2026 15:12:11 +0000 Subject: [PATCH 001/106] Switch to prek --- .github/workflows/lint.yml | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3f9c3e1..4153cad 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,16 +1,10 @@ -name: Reusable Lint Workflow - -on: - workflow_call: - inputs: - clang-format-version: - description: 'Version of clang-format to use' - required: false - type: string - default: '' +# Requires +# - pyproject.toml with a group called "lint" which must include prek +# - A prek.toml file +name: Lint workflow for tskit-dev jobs: - pre-commit: + prek: name: Lint runs-on: ubuntu-24.04 steps: @@ -21,12 +15,14 @@ jobs: - uses: actions/checkout@v4.2.2 - - uses: actions/setup-python@v5.4.0 + - name: Install uv and set the python version + uses: astral-sh/setup-uv@v6 with: - python-version: '3.12' + python-version: 3.11 + version: "0.8.15" + + - name: Install Python dependencies + run: uv sync --locked --only-group lint - - name: Install clang-format (if needed) - if: ${{ inputs.clang-format-version != '' }} - run: pip install clang-format==${{ inputs.clang-format-version }} - - - uses: pre-commit/action@v3.0.1 + - name: Run prek + run: uv run --only-group lint prek -c prek.toml run --all-files From f4ddcb07bc161f3f9cc8578db298a375631177e8 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Tue, 17 Feb 2026 15:18:41 +0000 Subject: [PATCH 002/106] Add workflow_call trigger to lint workflow --- .github/workflows/lint.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4153cad..6568893 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,6 +3,9 @@ # - A prek.toml file name: Lint workflow for tskit-dev +on: + workflow_call: + jobs: prek: name: Lint From 22abbc2fa6fbd8817ccaf1cc411cc200acf5d20f Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 19 Feb 2026 11:19:02 +0000 Subject: [PATCH 003/106] Always use --no-default-groups --- .github/workflows/docs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 245aae3..6e78bdc 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -58,7 +58,7 @@ jobs: version: "0.8.15" - name: Install doc deps - run: uv sync --project=${{ inputs.pyproject-directory }} --locked --group docs + run: uv sync --no-default-groups --project=${{ inputs.pyproject-directory }} --locked --group docs # This is complicated. We run an embedded python script using uv which # reads the project name from pyproject.toml, and then uses python's @@ -70,7 +70,7 @@ jobs: set -euo pipefail export PROJECT_DIR="${{ inputs.pyproject-directory }}" PKG_VERSION="$( - uv run --project "$PROJECT_DIR" python - < Date: Thu, 19 Feb 2026 11:39:10 +0000 Subject: [PATCH 004/106] Use specific dep group --- .github/workflows/python-c-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index 807f0f9..c0e5792 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -38,13 +38,13 @@ jobs: version: "0.8.15" - name: Install Python dependencies - run: uv sync --locked --group test + run: uv sync --locked --group python-c-test --no-default-groups - name: Build module - run: CFLAGS=--coverage uv run python setup.py build_ext --inplace + run: CFLAGS=--coverage uv run python --no-default-groups setup.py build_ext --inplace - name: Run low-level tests - run: uv run pytest -vs -n0 tests/test_lowlevel.py + run: uv run pytest --no-default-groups tests/test_lowlevel.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 From fdb667879ec05a210276f62245e1ec018634117d Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 19 Feb 2026 13:24:37 +0000 Subject: [PATCH 005/106] Fix command syntax in Python C tests workflow --- .github/workflows/python-c-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index c0e5792..c8e2926 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -41,7 +41,7 @@ jobs: run: uv sync --locked --group python-c-test --no-default-groups - name: Build module - run: CFLAGS=--coverage uv run python --no-default-groups setup.py build_ext --inplace + run: CFLAGS=--coverage uv run --no-default-groups python setup.py build_ext --inplace - name: Run low-level tests run: uv run pytest --no-default-groups tests/test_lowlevel.py From 2b5433dac6461289c9ae9258f3063e4257a274bb Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 19 Feb 2026 13:27:30 +0000 Subject: [PATCH 006/106] Rename standard ll test file --- .github/workflows/python-c-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index c8e2926..a2ed7f6 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -44,7 +44,7 @@ jobs: run: CFLAGS=--coverage uv run --no-default-groups python setup.py build_ext --inplace - name: Run low-level tests - run: uv run pytest --no-default-groups tests/test_lowlevel.py + run: uv run pytest --no-default-groups tests/test_python_c.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 From 0e152d9d43f612c078966fcd4658389029f5bdc6 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 19 Feb 2026 13:29:19 +0000 Subject: [PATCH 007/106] Fix command syntax for running low-level tests --- .github/workflows/python-c-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index a2ed7f6..3187a82 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -44,7 +44,7 @@ jobs: run: CFLAGS=--coverage uv run --no-default-groups python setup.py build_ext --inplace - name: Run low-level tests - run: uv run pytest --no-default-groups tests/test_python_c.py + run: uv run --no-default-groups pytest tests/test_python_c.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 From d7f44b1dd29f5fbc6a10f0796ff6099fb53b6e8d Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 19 Feb 2026 13:49:05 +0000 Subject: [PATCH 008/106] Update dependency group name in workflow --- .github/workflows/python-c-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index 3187a82..28b91e8 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -38,7 +38,7 @@ jobs: version: "0.8.15" - name: Install Python dependencies - run: uv sync --locked --group python-c-test --no-default-groups + run: uv sync --locked --group test --no-default-groups - name: Build module run: CFLAGS=--coverage uv run --no-default-groups python setup.py build_ext --inplace From 85b6a05046cab486753da1d361542d6c1dca6532 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 19 Feb 2026 16:14:53 +0000 Subject: [PATCH 009/106] Add show-diff-on-failure option to prek lint command --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6568893..b728565 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -28,4 +28,4 @@ jobs: run: uv sync --locked --only-group lint - name: Run prek - run: uv run --only-group lint prek -c prek.toml run --all-files + run: uv run --only-group lint prek -c prek.toml run --all-files --show-diff-on-failure From cad3cd9e4c40d7da5e93d5d67afa0727772b9e8f Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 20 Feb 2026 10:56:38 +0000 Subject: [PATCH 010/106] Add verbose flag to prek lint command --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b728565..757a1f5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -28,4 +28,4 @@ jobs: run: uv sync --locked --only-group lint - name: Run prek - run: uv run --only-group lint prek -c prek.toml run --all-files --show-diff-on-failure + run: uv run --only-group lint prek -c prek.toml run --all-files --show-diff-on-failure -vvv From 736de26123780a6ca0c445d045451563d38d993a Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 20 Feb 2026 13:43:15 +0000 Subject: [PATCH 011/106] fully lock uv run --- .github/workflows/lint.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 757a1f5..1ab7131 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -28,4 +28,6 @@ jobs: run: uv sync --locked --only-group lint - name: Run prek - run: uv run --only-group lint prek -c prek.toml run --all-files --show-diff-on-failure -vvv + run: | + uv run --locked --only-group lint \ + prek -c prek.toml run --all-files --show-diff-on-failure -vvv From dd87718f3b2d583e1ce2e3dc566a5f3df4e4c9b7 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 20 Feb 2026 13:59:09 +0000 Subject: [PATCH 012/106] Update uv version in lint workflow --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 1ab7131..d3f42fe 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,7 +22,7 @@ jobs: uses: astral-sh/setup-uv@v6 with: python-version: 3.11 - version: "0.8.15" + version: "0.10.2" - name: Install Python dependencies run: uv sync --locked --only-group lint From 87c5923f03070f67f72dacc1e99177084eedd6d5 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 20 Feb 2026 14:22:09 +0000 Subject: [PATCH 013/106] Use standard --group so package is installed --- .github/workflows/lint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d3f42fe..203928d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -25,9 +25,9 @@ jobs: version: "0.10.2" - name: Install Python dependencies - run: uv sync --locked --only-group lint + run: uv sync --locked --group lint --no-default-groups - name: Run prek run: | - uv run --locked --only-group lint \ + uv run --locked --no-default-groups --group lint \ prek -c prek.toml run --all-files --show-diff-on-failure -vvv From 67aa07be097cca1543554aa38996807c0e46bd52 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 20 Feb 2026 14:30:33 +0000 Subject: [PATCH 014/106] Revert back to group-only behaviour --- .github/workflows/lint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 203928d..4d38bb3 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -25,9 +25,9 @@ jobs: version: "0.10.2" - name: Install Python dependencies - run: uv sync --locked --group lint --no-default-groups + run: uv sync --locked --group-only lint - name: Run prek run: | - uv run --locked --no-default-groups --group lint \ + uv run --locked --group-only lint \ prek -c prek.toml run --all-files --show-diff-on-failure -vvv From 0bc4411a681bbceb162cbd1d4aedd8e27b34c44b Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 20 Feb 2026 14:32:02 +0000 Subject: [PATCH 015/106] Update lint workflow to use 'only-group' option --- .github/workflows/lint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4d38bb3..993f4f9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -25,9 +25,9 @@ jobs: version: "0.10.2" - name: Install Python dependencies - run: uv sync --locked --group-only lint + run: uv sync --locked --only-group lint - name: Run prek run: | - uv run --locked --group-only lint \ + uv run --locked --only-group lint \ prek -c prek.toml run --all-files --show-diff-on-failure -vvv From 957680778786fc80f0e88227264fe22321e7dad0 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 20 Feb 2026 14:34:51 +0000 Subject: [PATCH 016/106] checkout submodules --- .github/workflows/lint.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 993f4f9..265c0a1 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -5,7 +5,7 @@ name: Lint workflow for tskit-dev on: workflow_call: - + jobs: prek: name: Lint @@ -17,6 +17,8 @@ jobs: access_token: ${{ github.token }} - uses: actions/checkout@v4.2.2 + with: + submodules: true - name: Install uv and set the python version uses: astral-sh/setup-uv@v6 From 84420f030c66035b734d397331ce9e26862195a2 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 20 Feb 2026 14:37:21 +0000 Subject: [PATCH 017/106] Drop verbosity Remove verbosity from prek command in lint workflow. --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 265c0a1..facb741 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -32,4 +32,4 @@ jobs: - name: Run prek run: | uv run --locked --only-group lint \ - prek -c prek.toml run --all-files --show-diff-on-failure -vvv + prek -c prek.toml run --all-files --show-diff-on-failure From 77c64795f4428d377dd03da67a33c3fbac9c03fc Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 20 Feb 2026 14:50:45 +0000 Subject: [PATCH 018/106] Finalise lint workflow --- .github/workflows/lint.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index facb741..cec2873 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,7 +26,10 @@ jobs: python-version: 3.11 version: "0.10.2" - - name: Install Python dependencies + - name: Install lint group + # Nore there is some subtlety here in using the --only-group approach. + # We **do not** install the local package, which means we don't need + # any system dependencies, but it can lead to oddities. run: uv sync --locked --only-group lint - name: Run prek From a266645a88cf9d206c8c9def143a3aa721afc0c6 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Sat, 21 Feb 2026 09:52:06 +0000 Subject: [PATCH 019/106] Add input for pyproject directory in lint workflow --- .github/workflows/lint.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cec2873..2d54a1a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -5,6 +5,12 @@ name: Lint workflow for tskit-dev on: workflow_call: + inputs: + pyproject-directory: + description: 'Directory to find pyproject.toml' + required: false + type: string + default: '.' jobs: prek: @@ -30,9 +36,9 @@ jobs: # Nore there is some subtlety here in using the --only-group approach. # We **do not** install the local package, which means we don't need # any system dependencies, but it can lead to oddities. - run: uv sync --locked --only-group lint + run: uv sync --locked --project=${{ inputs.pyproject-directory }} --only-group lint - name: Run prek run: | - uv run --locked --only-group lint \ - prek -c prek.toml run --all-files --show-diff-on-failure + uv run --locked --project=${{ inputs.pyproject-directory }} --only-group lint \ + prek -c ${{ inputs.pyproject-directory }}/prek.toml run --all-files --show-diff-on-failure From 52836c17fe368439391e153082b432da5aacf47b Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Sat, 21 Feb 2026 10:06:10 +0000 Subject: [PATCH 020/106] Add pyproject-directory input to CI workflow --- .github/workflows/python-c-tests.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index 28b91e8..5000584 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -8,9 +8,14 @@ on: required: false type: string default: '' + pyproject-directory: + description: 'Directory to find pyproject.toml' + required: false + type: string + default: '.' jobs: - c-tests: + python-c-tests: name: Python-C unit tests runs-on: ubuntu-24.04 @@ -38,13 +43,15 @@ jobs: version: "0.8.15" - name: Install Python dependencies - run: uv sync --locked --group test --no-default-groups + run: uv sync --project=${{ inputs.pyproject-directory }} --locked --group test --no-default-groups - name: Build module - run: CFLAGS=--coverage uv run --no-default-groups python setup.py build_ext --inplace + run: | + CFLAGS=--coverage uv run --project=${{ inputs.pyproject-directory }} --no-default-groups \ + python setup.py build_ext --inplace - name: Run low-level tests - run: uv run --no-default-groups pytest tests/test_python_c.py + run: uv run --project=${{ inputs.pyproject-directory }} --no-default-groups pytest tests/test_python_c.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 From 800384bdd31aede5785a300aab027492eb744550 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Sat, 21 Feb 2026 10:10:59 +0000 Subject: [PATCH 021/106] Fix working directory for Python C tests in CI --- .github/workflows/python-c-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index 5000584..58ed59b 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -46,12 +46,12 @@ jobs: run: uv sync --project=${{ inputs.pyproject-directory }} --locked --group test --no-default-groups - name: Build module - run: | - CFLAGS=--coverage uv run --project=${{ inputs.pyproject-directory }} --no-default-groups \ - python setup.py build_ext --inplace + working-directory: ${{ inputs.pyproject-directory }} + run: CFLAGS=--coverage uv run -no-default-groups python setup.py build_ext --inplace - name: Run low-level tests - run: uv run --project=${{ inputs.pyproject-directory }} --no-default-groups pytest tests/test_python_c.py + working-directory: ${{ inputs.pyproject-directory }} + run: uv run --no-default-groups pytest tests/test_python_c.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 From b838c090df099bb3e122df7ac8f79d130d49b5b6 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Sat, 21 Feb 2026 10:21:07 +0000 Subject: [PATCH 022/106] Fix command syntax in Python C tests workflow --- .github/workflows/python-c-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index 58ed59b..bf47a67 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -47,7 +47,7 @@ jobs: - name: Build module working-directory: ${{ inputs.pyproject-directory }} - run: CFLAGS=--coverage uv run -no-default-groups python setup.py build_ext --inplace + run: CFLAGS=--coverage uv run --no-default-groups python setup.py build_ext --inplace - name: Run low-level tests working-directory: ${{ inputs.pyproject-directory }} From 4beda702b3862007c090f84629731655e542f851 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Sat, 21 Feb 2026 10:23:42 +0000 Subject: [PATCH 023/106] Set CC to gcc for Meson configuration --- .github/workflows/c-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/c-tests.yml b/.github/workflows/c-tests.yml index 85c8804..f96d75d 100644 --- a/.github/workflows/c-tests.yml +++ b/.github/workflows/c-tests.yml @@ -37,7 +37,7 @@ jobs: - name: Configure code working-directory: ${{ inputs.library-directory }} - run: meson setup build-cov -Db_coverage=true + run: CC=gcc meson setup build-cov -Db_coverage=true - name: Compile working-directory: ${{ inputs.library-directory }} From e33e8c2c592065e5152f15584b727f3bf1d882c8 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Sat, 21 Feb 2026 15:02:54 +0000 Subject: [PATCH 024/106] Install clang separately --- .github/workflows/c-tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/c-tests.yml b/.github/workflows/c-tests.yml index f96d75d..4588715 100644 --- a/.github/workflows/c-tests.yml +++ b/.github/workflows/c-tests.yml @@ -33,7 +33,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libcunit1-dev libconfig-dev \ - ninja-build valgrind clang meson gcovr ${{ inputs.additional-apt-packages }} + ninja-build valgrind meson gcovr ${{ inputs.additional-apt-packages }} - name: Configure code working-directory: ${{ inputs.library-directory }} @@ -60,6 +60,9 @@ jobs: gcov_args: --preserve-paths fail_ci_if_error: true + - name: Install clang + run: sudo apt-get install -y clang + - name: Compile and test with clang working-directory: ${{ inputs.library-directory }} run: | From eb1e9ede4529e684a9bf4e09f64f049ccd7040f4 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Sat, 21 Feb 2026 15:35:47 +0000 Subject: [PATCH 025/106] Force gcov in coverage report generation --- .github/workflows/c-tests.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/c-tests.yml b/.github/workflows/c-tests.yml index 4588715..5856c39 100644 --- a/.github/workflows/c-tests.yml +++ b/.github/workflows/c-tests.yml @@ -33,7 +33,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libcunit1-dev libconfig-dev \ - ninja-build valgrind meson gcovr ${{ inputs.additional-apt-packages }} + ninja-build valgrind clang meson gcovr ${{ inputs.additional-apt-packages }} - name: Configure code working-directory: ${{ inputs.library-directory }} @@ -49,7 +49,7 @@ jobs: - name: Build coverage report working-directory: ${{ inputs.library-directory }} - run: ninja -C build-cov coverage-xml + run: GCOV=gcov ninja -C build-cov coverage-xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 @@ -59,9 +59,6 @@ jobs: directory: ${{ inputs.library-directory }} gcov_args: --preserve-paths fail_ci_if_error: true - - - name: Install clang - run: sudo apt-get install -y clang - name: Compile and test with clang working-directory: ${{ inputs.library-directory }} From 184e15e39877bc9d92391960c928989620d8a00f Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Sat, 21 Feb 2026 16:06:12 +0000 Subject: [PATCH 026/106] Initial uv install --- .github/workflows/c-tests.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/c-tests.yml b/.github/workflows/c-tests.yml index 5856c39..1734b39 100644 --- a/.github/workflows/c-tests.yml +++ b/.github/workflows/c-tests.yml @@ -33,7 +33,15 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libcunit1-dev libconfig-dev \ - ninja-build valgrind clang meson gcovr ${{ inputs.additional-apt-packages }} + ninja-build valgrind clang ${{ inputs.additional-apt-packages }} + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.10.2" + + - name: Install uv deps + run: uv tool install gcovr meson - name: Configure code working-directory: ${{ inputs.library-directory }} @@ -49,7 +57,7 @@ jobs: - name: Build coverage report working-directory: ${{ inputs.library-directory }} - run: GCOV=gcov ninja -C build-cov coverage-xml + run: ninja -C build-cov coverage-xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 From 1940f128413847780b1c170dd8f22d74a23082e8 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Sat, 21 Feb 2026 16:08:39 +0000 Subject: [PATCH 027/106] Tweak --- .github/workflows/c-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/c-tests.yml b/.github/workflows/c-tests.yml index 1734b39..de65ff0 100644 --- a/.github/workflows/c-tests.yml +++ b/.github/workflows/c-tests.yml @@ -41,7 +41,7 @@ jobs: version: "0.10.2" - name: Install uv deps - run: uv tool install gcovr meson + run: uv tool install gcovr; uv tool install meson - name: Configure code working-directory: ${{ inputs.library-directory }} From ae88782fc45b981c315ab2ad003a0e8f09e4a375 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Sat, 21 Feb 2026 16:38:47 +0000 Subject: [PATCH 028/106] use pyproject and pin deps fully --- .github/workflows/c-tests.yml | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/.github/workflows/c-tests.yml b/.github/workflows/c-tests.yml index de65ff0..22f54f2 100644 --- a/.github/workflows/c-tests.yml +++ b/.github/workflows/c-tests.yml @@ -3,6 +3,11 @@ name: Reusable C unit tests on: workflow_call: inputs: + pyproject-directory: + description: 'Directory to find pyproject.toml' + required: false + type: string + default: '.' additional-apt-packages: description: 'Additional APT packages to install' required: false @@ -29,7 +34,7 @@ jobs: with: submodules: true - - name: Install dev tools + - name: Install system deps run: | sudo apt-get update sudo apt-get install -y libcunit1-dev libconfig-dev \ @@ -41,11 +46,13 @@ jobs: version: "0.10.2" - name: Install uv deps - run: uv tool install gcovr; uv tool install meson + run: uv sync --project=${{ inputs.pyproject-directory }} --locked --only-group cdev - name: Configure code working-directory: ${{ inputs.library-directory }} - run: CC=gcc meson setup build-cov -Db_coverage=true + run: | + CC=gcc uv run --locked --project=${{ inputs.pyproject-directory }} --only-group cdev \ + meson setup build-cov -Db_coverage=true - name: Compile working-directory: ${{ inputs.library-directory }} @@ -57,7 +64,9 @@ jobs: - name: Build coverage report working-directory: ${{ inputs.library-directory }} - run: ninja -C build-cov coverage-xml + run: | + GCOVR='uv run --locked --project=${{ inputs.pyproject-directory }} --only-group cdev gcovr'\ + ninja -C build-cov coverage-xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 @@ -71,12 +80,15 @@ jobs: - name: Compile and test with clang working-directory: ${{ inputs.library-directory }} run: | - CC=clang meson setup build-clang - meson test -C build-clang + CC=clang uv run --locked --project=${{ inputs.pyproject-directory }} --only-group cdev \ + meson setup build-clang + ninja -C build-clang test - name: Run tests with valgrind working-directory: ${{ inputs.library-directory }} run: | - meson setup build-valgrind - meson test -C build-valgrind -t-1 \ - --wrapper='valgrind --leak-check=full --error-exitcode=1' + uv run --locked --project=${{ inputs.pyproject-directory }} --only-group cdev \ + meson setup build-valgrind + uv run --locked --project=${{ inputs.pyproject-directory }} --only-group cdev \ + meson test -C build-valgrind -t-1 \ + --wrapper='valgrind --leak-check=full --error-exitcode=1' From f38723081eb3bacc5aba571ee90ce8d6d004e532 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Sat, 21 Feb 2026 16:41:34 +0000 Subject: [PATCH 029/106] Fix indentation in c-tests.yml --- .github/workflows/c-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/c-tests.yml b/.github/workflows/c-tests.yml index 22f54f2..8e76c89 100644 --- a/.github/workflows/c-tests.yml +++ b/.github/workflows/c-tests.yml @@ -3,7 +3,7 @@ name: Reusable C unit tests on: workflow_call: inputs: - pyproject-directory: + pyproject-directory: description: 'Directory to find pyproject.toml' required: false type: string From 5dc4bb98a590aa71533bcdf6676cbc6328533a00 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Sat, 21 Feb 2026 16:51:00 +0000 Subject: [PATCH 030/106] Move meson build to root --- .github/workflows/c-tests.yml | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/c-tests.yml b/.github/workflows/c-tests.yml index 8e76c89..ef3e81a 100644 --- a/.github/workflows/c-tests.yml +++ b/.github/workflows/c-tests.yml @@ -49,46 +49,40 @@ jobs: run: uv sync --project=${{ inputs.pyproject-directory }} --locked --only-group cdev - name: Configure code - working-directory: ${{ inputs.library-directory }} + run: | CC=gcc uv run --locked --project=${{ inputs.pyproject-directory }} --only-group cdev \ - meson setup build-cov -Db_coverage=true + meson setup -Db_coverage=true build-cov ${{ inputs.library-directory }} - name: Compile - working-directory: ${{ inputs.library-directory }} run: ninja -C build-cov - + - name: Run tests - working-directory: ${{ inputs.library-directory }} run: ninja -C build-cov test - name: Build coverage report - working-directory: ${{ inputs.library-directory }} run: | GCOVR='uv run --locked --project=${{ inputs.pyproject-directory }} --only-group cdev gcovr'\ - ninja -C build-cov coverage-xml + ninja -C build-cov coverage-xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 with: token: ${{ secrets.CODECOV_TOKEN }} flags: C - directory: ${{ inputs.library-directory }} gcov_args: --preserve-paths fail_ci_if_error: true - name: Compile and test with clang - working-directory: ${{ inputs.library-directory }} run: | CC=clang uv run --locked --project=${{ inputs.pyproject-directory }} --only-group cdev \ - meson setup build-clang - ninja -C build-clang test + meson setup build-clang ${{ inputs.library-directory }} + ninja -C build-clang test - name: Run tests with valgrind - working-directory: ${{ inputs.library-directory }} run: | uv run --locked --project=${{ inputs.pyproject-directory }} --only-group cdev \ - meson setup build-valgrind + meson setup build-valgrind ${{ inputs.library-directory }} uv run --locked --project=${{ inputs.pyproject-directory }} --only-group cdev \ meson test -C build-valgrind -t-1 \ --wrapper='valgrind --leak-check=full --error-exitcode=1' From 0a1ec7473cad96ec726d1e9d029bbd11d90bbd8f Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Sat, 21 Feb 2026 17:16:24 +0000 Subject: [PATCH 031/106] Pin explicitly to versions from uv for gcovr --- .github/workflows/c-tests.yml | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/.github/workflows/c-tests.yml b/.github/workflows/c-tests.yml index ef3e81a..b706657 100644 --- a/.github/workflows/c-tests.yml +++ b/.github/workflows/c-tests.yml @@ -3,11 +3,6 @@ name: Reusable C unit tests on: workflow_call: inputs: - pyproject-directory: - description: 'Directory to find pyproject.toml' - required: false - type: string - default: '.' additional-apt-packages: description: 'Additional APT packages to install' required: false @@ -46,13 +41,12 @@ jobs: version: "0.10.2" - name: Install uv deps - run: uv sync --project=${{ inputs.pyproject-directory }} --locked --only-group cdev + run: | + uv tool install meson==1.10.1 + uv tool install gcovr==8.6 - name: Configure code - - run: | - CC=gcc uv run --locked --project=${{ inputs.pyproject-directory }} --only-group cdev \ - meson setup -Db_coverage=true build-cov ${{ inputs.library-directory }} + run: CC=gcc meson setup -Db_coverage=true build-cov ${{ inputs.library-directory }} - name: Compile run: ninja -C build-cov @@ -61,9 +55,7 @@ jobs: run: ninja -C build-cov test - name: Build coverage report - run: | - GCOVR='uv run --locked --project=${{ inputs.pyproject-directory }} --only-group cdev gcovr'\ - ninja -C build-cov coverage-xml + run: ninja -C build-cov coverage-xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 @@ -75,14 +67,11 @@ jobs: - name: Compile and test with clang run: | - CC=clang uv run --locked --project=${{ inputs.pyproject-directory }} --only-group cdev \ - meson setup build-clang ${{ inputs.library-directory }} + CC=clang meson setup build-clang ${{ inputs.library-directory }} ninja -C build-clang test - name: Run tests with valgrind run: | - uv run --locked --project=${{ inputs.pyproject-directory }} --only-group cdev \ - meson setup build-valgrind ${{ inputs.library-directory }} - uv run --locked --project=${{ inputs.pyproject-directory }} --only-group cdev \ - meson test -C build-valgrind -t-1 \ + meson setup build-valgrind ${{ inputs.library-directory }} + meson test -C build-valgrind -t-1 \ --wrapper='valgrind --leak-check=full --error-exitcode=1' From 8e472fb9f4e53459e2d136e827069bb1888ab077 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Sat, 21 Feb 2026 17:27:56 +0000 Subject: [PATCH 032/106] preserve paths, drop gcovr --- .github/workflows/python-c-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index bf47a67..08c4c4f 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -33,8 +33,7 @@ jobs: - name: Install dev tools run: | sudo apt-get update - sudo apt-get install -y \ - gcovr ${{ inputs.additional-apt-packages }} + sudo apt-get install -y ${{ inputs.additional-apt-packages }} - name: Install uv and set the python version uses: astral-sh/setup-uv@v6 @@ -57,5 +56,6 @@ jobs: uses: codecov/codecov-action@v5.4.0 with: token: ${{ secrets.CODECOV_TOKEN }} + gcov_args: --preserve-paths flags: c-python fail_ci_if_error: true From 602bf379b0c98d22e7d73656389b3973ac88383d Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Mon, 23 Feb 2026 10:16:41 +0000 Subject: [PATCH 033/106] Require prek.toml in project root --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2d54a1a..b014d21 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -41,4 +41,4 @@ jobs: - name: Run prek run: | uv run --locked --project=${{ inputs.pyproject-directory }} --only-group lint \ - prek -c ${{ inputs.pyproject-directory }}/prek.toml run --all-files --show-diff-on-failure + prek -c prek.toml run --all-files --show-diff-on-failure From a457db2df52fb438d6f36fd0ef7701244be82f4b Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Mon, 23 Feb 2026 10:47:58 +0000 Subject: [PATCH 034/106] Run tests from root --- .github/workflows/python-c-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index 08c4c4f..7ae53ff 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -49,8 +49,8 @@ jobs: run: CFLAGS=--coverage uv run --no-default-groups python setup.py build_ext --inplace - name: Run low-level tests - working-directory: ${{ inputs.pyproject-directory }} - run: uv run --no-default-groups pytest tests/test_python_c.py + run: uv run --project=${{ inputs.pyproject-directory }} --no-default-groups \ + pytest ${{ inputs.pyproject-directory }}/tests/test_python_c.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 From 65cff9b92aeced2f5febc408cf7aac232d6955a7 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Mon, 23 Feb 2026 10:51:46 +0000 Subject: [PATCH 035/106] fix --- .github/workflows/python-c-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index 7ae53ff..4a6af66 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -49,7 +49,7 @@ jobs: run: CFLAGS=--coverage uv run --no-default-groups python setup.py build_ext --inplace - name: Run low-level tests - run: uv run --project=${{ inputs.pyproject-directory }} --no-default-groups \ + run: uv run --project=${{ inputs.pyproject-directory }} --no-default-groups --locked --group test \ pytest ${{ inputs.pyproject-directory }}/tests/test_python_c.py - name: Upload coverage to Codecov From 76b03d97ebe5f6efaac0572bb888a3a78b0cfbc7 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Mon, 23 Feb 2026 10:58:59 +0000 Subject: [PATCH 036/106] fixup --- .github/workflows/python-c-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index 4a6af66..5f14098 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -49,8 +49,8 @@ jobs: run: CFLAGS=--coverage uv run --no-default-groups python setup.py build_ext --inplace - name: Run low-level tests - run: uv run --project=${{ inputs.pyproject-directory }} --no-default-groups --locked --group test \ - pytest ${{ inputs.pyproject-directory }}/tests/test_python_c.py + run: uv run --project=${{ inputs.pyproject-directory }} --locked --group test --no-default-groups pytest \ + ${{ inputs.pyproject-directory }}/tests/test_python_c.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 From e1d6781c886db5c0580160c2519c80bc297232b5 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Mon, 23 Feb 2026 11:58:07 +0000 Subject: [PATCH 037/106] Tweak --- .github/workflows/python-c-tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index 5f14098..0b7a76a 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -49,8 +49,9 @@ jobs: run: CFLAGS=--coverage uv run --no-default-groups python setup.py build_ext --inplace - name: Run low-level tests - run: uv run --project=${{ inputs.pyproject-directory }} --locked --group test --no-default-groups pytest \ - ${{ inputs.pyproject-directory }}/tests/test_python_c.py + run: | + uv run --project=${{ inputs.pyproject-directory }} --locked --group test --no-default-groups \ + pytest ${{ inputs.pyproject-directory }}/tests/test_python_c.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 From 9455fa64c389daefeb1b31ff43fa6ffd844875e4 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Mon, 23 Feb 2026 13:14:32 +0000 Subject: [PATCH 038/106] Install apt package if needed only --- .github/workflows/python-c-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index 0b7a76a..591b9e7 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -31,9 +31,9 @@ jobs: submodules: true - name: Install dev tools + if: ${{ inputs.additional-apt-packages != '' }} run: | - sudo apt-get update - sudo apt-get install -y ${{ inputs.additional-apt-packages }} + sudo apt-get update && sudo apt-get install -y ${{ inputs.additional-apt-packages }} - name: Install uv and set the python version uses: astral-sh/setup-uv@v6 From c8b8b6604cef744c71a874d42a40e80751ebc99a Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Mon, 23 Feb 2026 14:00:45 +0000 Subject: [PATCH 039/106] Install gcovr and run manually --- .github/workflows/python-c-tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index 591b9e7..857993a 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -44,6 +44,9 @@ jobs: - name: Install Python dependencies run: uv sync --project=${{ inputs.pyproject-directory }} --locked --group test --no-default-groups + - name: Install gcovr + run: uv tool install gcovr==8.6 + - name: Build module working-directory: ${{ inputs.pyproject-directory }} run: CFLAGS=--coverage uv run --no-default-groups python setup.py build_ext --inplace @@ -53,6 +56,9 @@ jobs: uv run --project=${{ inputs.pyproject-directory }} --locked --group test --no-default-groups \ pytest ${{ inputs.pyproject-directory }}/tests/test_python_c.py + - name: Run gcov + run: uv tool run gcovr + - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 with: From 36fd4300e942bada779cae25e9667e543d95aacc Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Mon, 23 Feb 2026 14:12:44 +0000 Subject: [PATCH 040/106] Explicitly mark coverage file --- .github/workflows/python-c-tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index 857993a..a2b5c66 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -57,12 +57,13 @@ jobs: pytest ${{ inputs.pyproject-directory }}/tests/test_python_c.py - name: Run gcov - run: uv tool run gcovr + run: uv tool run gcovr -o coverage.txt - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 with: token: ${{ secrets.CODECOV_TOKEN }} - gcov_args: --preserve-paths + files: coverage.txt + verbose: true flags: c-python fail_ci_if_error: true From 99046836aef0a8a33c60433c28101361aad3818f Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Mon, 23 Feb 2026 14:18:54 +0000 Subject: [PATCH 041/106] Tweak --- .github/workflows/python-c-tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index a2b5c66..5557e66 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -56,14 +56,15 @@ jobs: uv run --project=${{ inputs.pyproject-directory }} --locked --group test --no-default-groups \ pytest ${{ inputs.pyproject-directory }}/tests/test_python_c.py - - name: Run gcov - run: uv tool run gcovr -o coverage.txt + - name: Run gcovr + run: uv tool run gcovr -o coverage.txt && cat coverage.txt - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 with: token: ${{ secrets.CODECOV_TOKEN }} files: coverage.txt + disable_search: true verbose: true flags: c-python fail_ci_if_error: true From 9a255c3912ea4c1de94d13175165bd97ac260910 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Mon, 23 Feb 2026 14:25:55 +0000 Subject: [PATCH 042/106] Switch to xml --- .github/workflows/python-c-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index 5557e66..953725f 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -57,13 +57,13 @@ jobs: pytest ${{ inputs.pyproject-directory }}/tests/test_python_c.py - name: Run gcovr - run: uv tool run gcovr -o coverage.txt && cat coverage.txt + run: uv tool run gcovr --clover coverage.xml --txt - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.4.0 with: token: ${{ secrets.CODECOV_TOKEN }} - files: coverage.txt + files: coverage.xml disable_search: true verbose: true flags: c-python From 59e4e136a1d411f00eec3f34d2d250c426039960 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Mon, 23 Feb 2026 14:42:01 +0000 Subject: [PATCH 043/106] Add input for low-level tests in workflow --- .github/workflows/python-c-tests.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index 953725f..d0319a2 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -13,6 +13,11 @@ on: required: false type: string default: '.' + tests: + description: 'Low-level tests to run' + required: false + type: string + default: 'tests/test_python_c.py' jobs: python-c-tests: @@ -54,7 +59,7 @@ jobs: - name: Run low-level tests run: | uv run --project=${{ inputs.pyproject-directory }} --locked --group test --no-default-groups \ - pytest ${{ inputs.pyproject-directory }}/tests/test_python_c.py + pytest ${{ inputs.tests }} - name: Run gcovr run: uv tool run gcovr --clover coverage.xml --txt From e7e9606c0a75774dc71c8562e260dcb7537a62f5 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Tue, 24 Feb 2026 14:38:20 +0000 Subject: [PATCH 044/106] Add general Python test workflow --- .github/workflows/python-tests.yml | 62 +++++++++++------------------- 1 file changed, 22 insertions(+), 40 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 0067aac..9d56c56 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -3,16 +3,6 @@ name: Reusable Python Tests on: workflow_call: inputs: - additional_commands: - description: 'Additional commands to run after installing dependencies' - required: false - type: string - default: '' - make_command: - description: 'Make/build command to run' - required: false - type: string - default: '' python_version: description: 'Python version to use' required: true @@ -22,6 +12,18 @@ on: required: true type: string + pyproject-directory: + description: 'Directory to find pyproject.toml' + required: false + type: string + default: '.' + + additional-apt-packages: + description: 'Additional APT packages to install' + required: false + type: string + default: '' + jobs: test: name: Python @@ -40,48 +42,28 @@ jobs: with: submodules: true - - name: Determine working directory - id: workdir - run: | - if [ -d "python" ]; then - echo "dir=python" >> $GITHUB_OUTPUT - else - echo "dir=." >> $GITHUB_OUTPUT - fi + - name: Install additional APT packages + if: ${{ inputs.additional-apt-packages != '' }} + run: sudo apt-get update && sudo apt-get install -y ${{ inputs.additional-apt-packages }} - name: Install uv and set the python version uses: astral-sh/setup-uv@v6 with: python-version: ${{ inputs.python_version }} version: "0.8.15" - - - name: Install dependencies - working-directory: ${{ steps.workdir.outputs.dir }} - run: | - uv venv - uv pip install -r pyproject.toml --extra test - - - name: Additional commands - if: ${{ inputs.additional_commands != '' }} - working-directory: ${{ steps.workdir.outputs.dir }} - run: ${{ inputs.additional_commands }} - - name: Make/build - if: ${{ inputs.make_command != '' }} - working-directory: ${{ steps.workdir.outputs.dir }} - run: ${{ inputs.make_command }} + - name: Install test deps + run: uv sync --project=${{ inputs.pyproject-directory }} --locked --group tests - name: Run tests - working-directory: ${{ steps.workdir.outputs.dir }} - run: | - uv run --no-sync python -m pytest -xv --cov --cov-report=xml --cov-branch + run: uv run --project=${{ inputs.pyproject-directory }} --cov-report=xml --cov-branch -n2 - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5.4.0 + uses: codecov/codecov-action@v5.5.2 with: token: ${{ secrets.CODECOV_TOKEN }} - working-directory: ${{ steps.workdir.outputs.dir }} fail_ci_if_error: false flags: python-tests - name: codecov-umbrella - verbose: true \ No newline at end of file + files: coverage.xml + disable_search: true + verbose: true From 8242cb8f7bcfd5bc46b7c9867617d9be089f77cf Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Tue, 24 Feb 2026 14:43:37 +0000 Subject: [PATCH 045/106] Fix input key for Python version in workflow --- .github/workflows/python-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 9d56c56..ccea605 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -3,7 +3,7 @@ name: Reusable Python Tests on: workflow_call: inputs: - python_version: + python-version: description: 'Python version to use' required: true type: string @@ -49,7 +49,7 @@ jobs: - name: Install uv and set the python version uses: astral-sh/setup-uv@v6 with: - python-version: ${{ inputs.python_version }} + python-version: ${{ inputs.python-version }} version: "0.8.15" - name: Install test deps From d3583f5cda0e4d094a0bb0c179003c522f446c3d Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Tue, 24 Feb 2026 14:46:06 +0000 Subject: [PATCH 046/106] fixups Updated test group name and adjusted test run command. --- .github/workflows/python-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index ccea605..88873b9 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -11,13 +11,11 @@ on: description: 'Operating system to run on' required: true type: string - pyproject-directory: description: 'Directory to find pyproject.toml' required: false type: string default: '.' - additional-apt-packages: description: 'Additional APT packages to install' required: false @@ -53,10 +51,12 @@ jobs: version: "0.8.15" - name: Install test deps - run: uv sync --project=${{ inputs.pyproject-directory }} --locked --group tests + run: uv sync --project=${{ inputs.pyproject-directory }} --locked --group test --no-default-groups - name: Run tests - run: uv run --project=${{ inputs.pyproject-directory }} --cov-report=xml --cov-branch -n2 + run: | + uv run --no-default-groups --project=${{ inputs.pyproject-directory }} + pytest --cov-report=xml --cov-branch -n2 - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.5.2 From c1c2340286f29812e7d9a39f6abfecac8d35cb2f Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Tue, 24 Feb 2026 14:46:46 +0000 Subject: [PATCH 047/106] Update test command to include locked option --- .github/workflows/python-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 88873b9..17fa09b 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -55,7 +55,7 @@ jobs: - name: Run tests run: | - uv run --no-default-groups --project=${{ inputs.pyproject-directory }} + uv run --project=${{ inputs.pyproject-directory }} --locked --group test --no-default-groups \ pytest --cov-report=xml --cov-branch -n2 - name: Upload coverage to Codecov From 5d8e737047fe2a83895398bf371dd2b6d6809a42 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Tue, 24 Feb 2026 15:03:37 +0000 Subject: [PATCH 048/106] Add coverage-directory --- .github/workflows/python-tests.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 17fa09b..3f85ba7 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -16,6 +16,10 @@ on: required: false type: string default: '.' + coverage-directory: + description: 'Directory to calculate coverage for' + required: true + type: string additional-apt-packages: description: 'Additional APT packages to install' required: false @@ -56,7 +60,7 @@ jobs: - name: Run tests run: | uv run --project=${{ inputs.pyproject-directory }} --locked --group test --no-default-groups \ - pytest --cov-report=xml --cov-branch -n2 + pytest --cov-report=xml --cov-branch -n2 --cov=${{ inputs.coverage-directory }} - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.5.2 From e3dbcca0f6af48cf5eadcddca9d9e5c50347df53 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Tue, 24 Feb 2026 15:05:47 +0000 Subject: [PATCH 049/106] Change Codecov action to fail CI on error --- .github/workflows/python-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 3f85ba7..c80c420 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -66,7 +66,7 @@ jobs: uses: codecov/codecov-action@v5.5.2 with: token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: false + fail_ci_if_error: true flags: python-tests files: coverage.xml disable_search: true From b257ad4026637aa688a5a8e179b5bfe148f24ccf Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Tue, 24 Feb 2026 15:06:43 +0000 Subject: [PATCH 050/106] Rename job from 'Python' to 'Unit tests' --- .github/workflows/python-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index c80c420..3d6d8fa 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -28,7 +28,7 @@ on: jobs: test: - name: Python + name: Unit tests runs-on: ${{ inputs.os }} defaults: run: From 4f2265c4a371773eef67dbbdd86a91fc3520e8f1 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Tue, 24 Feb 2026 15:52:19 +0000 Subject: [PATCH 051/106] Explicitly set test directory --- .github/workflows/python-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 3d6d8fa..7bff252 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -60,7 +60,8 @@ jobs: - name: Run tests run: | uv run --project=${{ inputs.pyproject-directory }} --locked --group test --no-default-groups \ - pytest --cov-report=xml --cov-branch -n2 --cov=${{ inputs.coverage-directory }} + pytest --cov-report=xml --cov-branch -n2 --cov=${{ inputs.coverage-directory }} \ + ${{ inputs.pyproject-directory }}/tests - name: Upload coverage to Codecov uses: codecov/codecov-action@v5.5.2 From eb5070f7821d4a39b7ead22feb2493f9b7c41c4f Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Tue, 24 Feb 2026 16:24:35 +0000 Subject: [PATCH 052/106] Remove additional-apt-packages from python-tests.yml Removed additional-apt-packages input from workflow. --- .github/workflows/python-tests.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 7bff252..5d0c830 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -20,11 +20,6 @@ on: description: 'Directory to calculate coverage for' required: true type: string - additional-apt-packages: - description: 'Additional APT packages to install' - required: false - type: string - default: '' jobs: test: From ab9823b37ce590a1887e9063aac89f16592c9a07 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Wed, 25 Feb 2026 09:51:16 +0000 Subject: [PATCH 053/106] Update packaging tests to use uv --- .github/workflows/python-packaging.yml | 42 +++++++++++++++----------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/.github/workflows/python-packaging.yml b/.github/workflows/python-packaging.yml index 5b7e2f1..48bae21 100644 --- a/.github/workflows/python-packaging.yml +++ b/.github/workflows/python-packaging.yml @@ -8,8 +8,8 @@ on: required: false type: string default: '' - working-directory: - description: 'Working directory to use (default if unspecified)' + pyproject-directory: + description: 'Directory to find pyproject.toml' required: false type: string default: '.' @@ -36,29 +36,37 @@ jobs: if: ${{ inputs.additional-apt-packages != '' }} run: sudo apt-get update && sudo apt-get install -y ${{ inputs.additional-apt-packages }} - - uses: actions/setup-python@v5.4.0 + - name: Install uv and set the python version + uses: astral-sh/setup-uv@v6 with: - python-version: '3.12' + python-version: "3.12" + version: "0.8.15" - - name: Install build deps - run: | - pip install uv - uv pip install --system build twine validate-pyproject[all] + - name: Install packaging deps + working-directory: ${{ inputs.pyproject-directory }} + run: uv sync --locked --group packaging --no-default-groups - name: Validate pyproject - working-directory: ${{ inputs.working-directory }} - run: validate-pyproject pyproject.toml + working-directory: ${{ inputs.pyproject-directory }} + run: uv run --locked --group packaging --no-default-groups validate-pyproject pyproject.toml -v + + - name: Build artefacts + working-directory: ${{ inputs.pyproject-directory }} + run: uv build - name: Twine check - working-directory: ${{ inputs.working-directory }} - run: | - python -m build - python -m twine check --strict dist/* + working-directory: ${{ inputs.pyproject-directory }} + run: uv run --locked --group packaging --no-default-groups twine check --strict dist/* - - name: Install + - name: Install into venv working-directory: ${{ inputs.working-directory }} - run: python -m pip install dist/*.whl + run: | + uv venv cli-test + uv pip install --python=cli-test dist/*.whl - name: Run CLI (if present) if: ${{ inputs.cli-test-cmd != '' }} - run: ${{ inputs.cli-test-cmd }} + working-directory: ${{ inputs.pyproject-directory }} + run: | + source cli-test/bin/activate + ./cli-test/bin/${{ inputs.cli-test-cmd }} From 33f2f5b6fe33c011c75304b488ff43c13921789e Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Wed, 25 Feb 2026 12:19:12 +0000 Subject: [PATCH 054/106] fetch tags to get correct package versions --- .github/workflows/python-packaging.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-packaging.yml b/.github/workflows/python-packaging.yml index 48bae21..be98e52 100644 --- a/.github/workflows/python-packaging.yml +++ b/.github/workflows/python-packaging.yml @@ -31,7 +31,8 @@ jobs: - uses: actions/checkout@v4.2.2 with: submodules: true - + fetch-tags: true + - name: Install additional APT packages if: ${{ inputs.additional-apt-packages != '' }} run: sudo apt-get update && sudo apt-get install -y ${{ inputs.additional-apt-packages }} From 81708c6c64e1b1d6ecb06219c7f7d5665fe00508 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Wed, 25 Feb 2026 12:22:20 +0000 Subject: [PATCH 055/106] use fetch-depth = 0 for setuptools_scm --- .github/workflows/python-packaging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-packaging.yml b/.github/workflows/python-packaging.yml index be98e52..1813627 100644 --- a/.github/workflows/python-packaging.yml +++ b/.github/workflows/python-packaging.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/checkout@v4.2.2 with: submodules: true - fetch-tags: true + fetch-depth: 0 - name: Install additional APT packages if: ${{ inputs.additional-apt-packages != '' }} From 4daed300dcec260a0b30418f27c8e0457262adce Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Wed, 25 Feb 2026 12:31:30 +0000 Subject: [PATCH 056/106] Initial publish workflow --- .github/workflows/publish.yml | 70 +++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..8935c1b --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,70 @@ +# Requires +# - pyproject.toml +name: Package publication for Python only projects + +on: + workflow_call: + inputs: + pyproject-directory: + description: 'Directory to find pyproject.toml' + required: false + type: string + default: '.' + +jobs: + build-distribution-artifacts: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4.2.2 + with: + fetch-depth: 0 + submodules: true + + - name: Install uv and set the python version + uses: astral-sh/setup-uv@v6 + with: + python-version: "3.12" + version: "0.8.15" + + - name: Build artifacts + working-directory: ${{ inputs.pyproject-directory }} + run: uv build + + - name: Upload artifacts + working-directory: ${{ inputs.pyproject-directory }} + uses: actions/upload-artifact@v6.0.0 + with: + name: dist + path: dist/ + if-no-files-found: error + + publish: + runs-on: ubuntu-24.04 + environment: release + needs: [ 'build-distribution-artifacts' ] + permissions: + id-token: write + steps: + - name: Download artifacts + working-directory: ${{ inputs.pyproject-directory }} + uses: actions/download-artifact@v7.0.0 + with: + name: dist + path: dist + + - name: Show artifacts + working-directory: ${{ inputs.pyproject-directory }} + run: ls -lah dist + + - name: Publish distribution to Test PyPI + working-directory: ${{ inputs.pyproject-directory }} + if: github.event_name == 'push' && github.ref_name == 'test-publish' + uses: pypa/gh-action-pypi-publish@v1.13.0 + with: + repository-url: https://test.pypi.org/legacy/ + verbose: true + + - name: Publish distribution to PRODUCTION PyPI + working-directory: ${{ inputs.pyproject-directory }} + if: github.event_name == 'release' + uses: pypa/gh-action-pypi-publish@v1.13.0 From 8f2d49f4dafdab7f1f2b208fac5e7e79ee213046 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Wed, 25 Feb 2026 12:45:24 +0000 Subject: [PATCH 057/106] fixup paths --- .github/workflows/publish.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8935c1b..951df4c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -31,11 +31,10 @@ jobs: run: uv build - name: Upload artifacts - working-directory: ${{ inputs.pyproject-directory }} uses: actions/upload-artifact@v6.0.0 with: name: dist - path: dist/ + path: ${{ inputs.pyproject-directory }}/dist/ if-no-files-found: error publish: @@ -46,7 +45,6 @@ jobs: id-token: write steps: - name: Download artifacts - working-directory: ${{ inputs.pyproject-directory }} uses: actions/download-artifact@v7.0.0 with: name: dist @@ -57,7 +55,6 @@ jobs: run: ls -lah dist - name: Publish distribution to Test PyPI - working-directory: ${{ inputs.pyproject-directory }} if: github.event_name == 'push' && github.ref_name == 'test-publish' uses: pypa/gh-action-pypi-publish@v1.13.0 with: @@ -65,6 +62,5 @@ jobs: verbose: true - name: Publish distribution to PRODUCTION PyPI - working-directory: ${{ inputs.pyproject-directory }} if: github.event_name == 'release' uses: pypa/gh-action-pypi-publish@v1.13.0 From 8b38617577aaa392248a8e7eb644eabdb61d04c4 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Wed, 25 Feb 2026 12:49:35 +0000 Subject: [PATCH 058/106] fixup paths --- .github/workflows/publish.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 951df4c..8971e49 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -27,14 +27,13 @@ jobs: version: "0.8.15" - name: Build artifacts - working-directory: ${{ inputs.pyproject-directory }} - run: uv build + run: uv build --project ${{ inputs.pyproject-directory }} -o dist - name: Upload artifacts uses: actions/upload-artifact@v6.0.0 with: name: dist - path: ${{ inputs.pyproject-directory }}/dist/ + path: dist/ if-no-files-found: error publish: From 5411c1f19f0c760efe48aca5de7ca7923555a72c Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Wed, 25 Feb 2026 12:49:55 +0000 Subject: [PATCH 059/106] Rename publish.yml to publish-python.yml --- .github/workflows/{publish.yml => publish-python.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{publish.yml => publish-python.yml} (100%) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish-python.yml similarity index 100% rename from .github/workflows/publish.yml rename to .github/workflows/publish-python.yml From d8109bf4065525f6f9c7de912f2f0614d5a2c93f Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Wed, 25 Feb 2026 14:11:38 +0000 Subject: [PATCH 060/106] Delete .github/workflows/publish-python.yml --- .github/workflows/publish-python.yml | 65 ---------------------------- 1 file changed, 65 deletions(-) delete mode 100644 .github/workflows/publish-python.yml diff --git a/.github/workflows/publish-python.yml b/.github/workflows/publish-python.yml deleted file mode 100644 index 8971e49..0000000 --- a/.github/workflows/publish-python.yml +++ /dev/null @@ -1,65 +0,0 @@ -# Requires -# - pyproject.toml -name: Package publication for Python only projects - -on: - workflow_call: - inputs: - pyproject-directory: - description: 'Directory to find pyproject.toml' - required: false - type: string - default: '.' - -jobs: - build-distribution-artifacts: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4.2.2 - with: - fetch-depth: 0 - submodules: true - - - name: Install uv and set the python version - uses: astral-sh/setup-uv@v6 - with: - python-version: "3.12" - version: "0.8.15" - - - name: Build artifacts - run: uv build --project ${{ inputs.pyproject-directory }} -o dist - - - name: Upload artifacts - uses: actions/upload-artifact@v6.0.0 - with: - name: dist - path: dist/ - if-no-files-found: error - - publish: - runs-on: ubuntu-24.04 - environment: release - needs: [ 'build-distribution-artifacts' ] - permissions: - id-token: write - steps: - - name: Download artifacts - uses: actions/download-artifact@v7.0.0 - with: - name: dist - path: dist - - - name: Show artifacts - working-directory: ${{ inputs.pyproject-directory }} - run: ls -lah dist - - - name: Publish distribution to Test PyPI - if: github.event_name == 'push' && github.ref_name == 'test-publish' - uses: pypa/gh-action-pypi-publish@v1.13.0 - with: - repository-url: https://test.pypi.org/legacy/ - verbose: true - - - name: Publish distribution to PRODUCTION PyPI - if: github.event_name == 'release' - uses: pypa/gh-action-pypi-publish@v1.13.0 From 7277d32c957bb89e9d00857809ae378a712905cb Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Wed, 25 Feb 2026 14:12:38 +0000 Subject: [PATCH 061/106] fix typo --- .github/workflows/python-packaging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-packaging.yml b/.github/workflows/python-packaging.yml index 1813627..b265bdb 100644 --- a/.github/workflows/python-packaging.yml +++ b/.github/workflows/python-packaging.yml @@ -60,7 +60,7 @@ jobs: run: uv run --locked --group packaging --no-default-groups twine check --strict dist/* - name: Install into venv - working-directory: ${{ inputs.working-directory }} + working-directory: ${{ inputs.pyproject-directory }} run: | uv venv cli-test uv pip install --python=cli-test dist/*.whl From 3063643b18b58f6fbeca22c8f6f88f2c1f62aa8e Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 26 Feb 2026 10:03:51 +0000 Subject: [PATCH 062/106] Initial addition of wheel build --- .github/workflows/build-wheels.yml | 83 ++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 .github/workflows/build-wheels.yml diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml new file mode 100644 index 0000000..feb4642 --- /dev/null +++ b/.github/workflows/build-wheels.yml @@ -0,0 +1,83 @@ +name: Binary wheel builds + +on: + workflow_call: + inputs: + pyproject-directory: + description: 'Directory to find pyproject.toml' + required: false + type: string + default: '.' + +jobs: + build-sdist: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + submodules: true + + - name: Install uv and set the python version + uses: astral-sh/setup-uv@v6 + with: + python-version: "3.12" + version: "0.8.15" + + - name: Build sdist + run: uv build --project=${{ inputs.pyproject-directory }} --sdist -o dist + + - name: Upload artifacts + uses: actions/upload-artifact@v6.0.0 + with: + name: build-sdist + path: dist/*.tar.gz + if-no-files-found: error + + build-wheels: + needs: [ 'build-sdist' ] + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest, macos-15-intel, macos-14] + steps: + - name: Cancel Previous Runs + uses: styfle/cancel-workflow-action@0.12.1 + with: + access_token: ${{ github.token }} + + - uses: actions/checkout@v5 + + - name: Download artifacts + uses: actions/download-artifact@v7.0.0 + with: + name: build-sdist + + - name: Copy sdist (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: Copy-Item *.tar.gz sdist.tar.gz + + - name: Copy sdist (Unix) + if: runner.os != 'Windows' + shell: bash + run: cp *.tar.gz sdist.tar.gz + + - name: Install uv and set the python version + uses: astral-sh/setup-uv@v6 + with: + python-version: "3.12" + version: "0.8.15" + + - name: Install cibuildwheel + run: uv sync --project=${{ inputs.pyproject-directory }} --locked --no-default-groups --group wheels + + - name: Build wheels + run: uv run --no-default-groups --project={{ inputs.pyproject-directory }} cibuildwheel sdist.tar.gz + + - uses: actions/upload-artifact@v4 + with: + name: build-wheels-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl + if-no-files-found: error From 6cfae10e10e4664c014f81303fc31b4d1b373cdf Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 26 Feb 2026 10:12:18 +0000 Subject: [PATCH 063/106] Update build-wheels.yml --- .github/workflows/build-wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index feb4642..ac320e6 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -74,7 +74,7 @@ jobs: run: uv sync --project=${{ inputs.pyproject-directory }} --locked --no-default-groups --group wheels - name: Build wheels - run: uv run --no-default-groups --project={{ inputs.pyproject-directory }} cibuildwheel sdist.tar.gz + run: uv run --no-default-groups --project=${{ inputs.pyproject-directory }} cibuildwheel sdist.tar.gz - uses: actions/upload-artifact@v4 with: From f7f3999226b97cbfdbb98dd32393354ff48a36ef Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 26 Feb 2026 11:07:44 +0000 Subject: [PATCH 064/106] Update build-wheels.yml --- .github/workflows/build-wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index ac320e6..a18c54d 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -71,10 +71,10 @@ jobs: version: "0.8.15" - name: Install cibuildwheel - run: uv sync --project=${{ inputs.pyproject-directory }} --locked --no-default-groups --group wheels + run: uv sync --project=${{ inputs.pyproject-directory }} --locked --only-group wheels - name: Build wheels - run: uv run --no-default-groups --project=${{ inputs.pyproject-directory }} cibuildwheel sdist.tar.gz + run: uv run --only-group wheels --project=${{ inputs.pyproject-directory }} cibuildwheel sdist.tar.gz - uses: actions/upload-artifact@v4 with: From 2f545581ce955de85e70640313ea18d191dd348e Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 26 Feb 2026 11:35:30 +0000 Subject: [PATCH 065/106] Make os-list a parameter --- .github/workflows/build-wheels.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index a18c54d..bc731dd 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -8,6 +8,11 @@ on: required: false type: string default: '.' + os-list: + description: "JSON array of OS runners" + required: false + type: string + default: '["ubuntu-24.04", "windows-2025", "macos-15-intel", "macos-14"]' jobs: build-sdist: @@ -40,7 +45,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macos-15-intel, macos-14] + os: ${{ fromJSON(inputs.os-list) }} steps: - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@0.12.1 From f8e0e36b7440ebb41759dab6de48ac84abaef2fb Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 26 Feb 2026 12:17:52 +0000 Subject: [PATCH 066/106] Only build native wheels on macOS --- .github/workflows/build-wheels.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index bc731dd..04f8f7d 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -77,7 +77,17 @@ jobs: - name: Install cibuildwheel run: uv sync --project=${{ inputs.pyproject-directory }} --locked --only-group wheels + + - name: macOS Intel: build x86_64 only + if: matrix.os == 'macos-15-intel' + shell: bash + run: echo "CIBW_ARCHS_MACOS=x86_64" >> "$GITHUB_ENV" + - name: macOS ARM: build arm64 only + if: matrix.os == 'macos-14' + shell: bash + run: echo "CIBW_ARCHS_MACOS=arm64" >> "$GITHUB_ENV" + - name: Build wheels run: uv run --only-group wheels --project=${{ inputs.pyproject-directory }} cibuildwheel sdist.tar.gz From 55090c8024eceabb62c74b631c0a8b9357d39cf9 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 26 Feb 2026 12:20:37 +0000 Subject: [PATCH 067/106] Update build-wheels.yml --- .github/workflows/build-wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 04f8f7d..d39342f 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -78,12 +78,12 @@ jobs: - name: Install cibuildwheel run: uv sync --project=${{ inputs.pyproject-directory }} --locked --only-group wheels - - name: macOS Intel: build x86_64 only + - name: macOS Intel if: matrix.os == 'macos-15-intel' shell: bash run: echo "CIBW_ARCHS_MACOS=x86_64" >> "$GITHUB_ENV" - - name: macOS ARM: build arm64 only + - name: macOS ARM if: matrix.os == 'macos-14' shell: bash run: echo "CIBW_ARCHS_MACOS=arm64" >> "$GITHUB_ENV" From d5ef16814130a15bf6c8247d5e5ff4714159f1ec Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 26 Feb 2026 12:59:11 +0000 Subject: [PATCH 068/106] Remove macos specific environment --- .github/workflows/build-wheels.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index d39342f..90b3a8a 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -77,17 +77,7 @@ jobs: - name: Install cibuildwheel run: uv sync --project=${{ inputs.pyproject-directory }} --locked --only-group wheels - - - name: macOS Intel - if: matrix.os == 'macos-15-intel' - shell: bash - run: echo "CIBW_ARCHS_MACOS=x86_64" >> "$GITHUB_ENV" - - - name: macOS ARM - if: matrix.os == 'macos-14' - shell: bash - run: echo "CIBW_ARCHS_MACOS=arm64" >> "$GITHUB_ENV" - + - name: Build wheels run: uv run --only-group wheels --project=${{ inputs.pyproject-directory }} cibuildwheel sdist.tar.gz From ab930f1af8cdcf660a51eafcffae21c137b71563 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 26 Feb 2026 13:05:54 +0000 Subject: [PATCH 069/106] Update build-wheels.yml --- .github/workflows/build-wheels.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 90b3a8a..393fb50 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -77,9 +77,25 @@ jobs: - name: Install cibuildwheel run: uv sync --project=${{ inputs.pyproject-directory }} --locked --only-group wheels - + + - name: Set CIBW_ARCHS_MACOS based on native arch + if: runner.os == 'macOS' + shell: bash + run: | + ARCH=$(uname -m) + if [ "$ARCH" = "x86_64" ]; then + echo "CIBW_ARCHS_MACOS=x86_64" >> "$GITHUB_ENV" + elif [ "$ARCH" = "arm64" ]; then + echo "CIBW_ARCHS_MACOS=arm64" >> "$GITHUB_ENV" + else + echo "Unknown macOS architecture: $ARCH" + exit 1 + fi + - name: Build wheels - run: uv run --only-group wheels --project=${{ inputs.pyproject-directory }} cibuildwheel sdist.tar.gz + run: | + uv run --only-group wheels --project=${{ inputs.pyproject-directory }} \ + cibuildwheel sdist.tar.gz - uses: actions/upload-artifact@v4 with: From e7d6e87e2a8a94c5a805bd38d123a3354db2e7d3 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 26 Feb 2026 13:15:21 +0000 Subject: [PATCH 070/106] Update build-wheels.yml --- .github/workflows/build-wheels.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 393fb50..bd58b74 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -93,9 +93,7 @@ jobs: fi - name: Build wheels - run: | - uv run --only-group wheels --project=${{ inputs.pyproject-directory }} \ - cibuildwheel sdist.tar.gz + run: uv run --only-group wheels --project=${{ inputs.pyproject-directory }} cibuildwheel sdist.tar.gz - uses: actions/upload-artifact@v4 with: From b3c5674e480df66536055a858442059d33d5976f Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 19 Feb 2026 16:44:07 +0000 Subject: [PATCH 071/106] Initial draft of repo administration doc --- repo_administration.md | 59 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 repo_administration.md diff --git a/repo_administration.md b/repo_administration.md new file mode 100644 index 0000000..50138ad --- /dev/null +++ b/repo_administration.md @@ -0,0 +1,59 @@ +# Repo administration + +This document describes the standard layout and organisation principles for +core tskit-dev projects. The GitHub workflows defined in this repo require +that this layout is followed. + +## Dependency management + +We use uv exclusively for Python package management. All Python dependencies must +be declared explicitly in pyproject.toml using dependency groups, which map to tasks +like running python tests or building documentation. The dependencies listed should +be minimal, and pinned only where necessary. + +## Test coverage + +Test coverage is monitored by CodeCov. When tests are run in CI, coverage is computed +and uploaded to CodeCov for each PR (and on merge). This is done using the +``CODECOV_TOKEN`` secret, which is generated by CodeCov and associated with a +given repo by GitHub. The token must be generated by logging in to the CodeCov UI, +copying the token and pasting it into the GitHub UI. This is in the settings/secrets +and variables/Actions section (repository secrets). It is important to check that +uploads are working correctly - do this by examining the GitHub actions logs and looking +at the output of the CodeCov upload command. + +## Standard workflows + +### Docs + +All documentation is built using jupyterbook and is defined in the top-level "docs" +directory. The ``docs.yml`` workflow builds documentation and incorporates it +into the tskit.dev website. Two versions of the documentation are maintained, +"latest" and "stable". The latest version is updated from the repo each time +there is a merge. The stable version is updated on release. + + +### Lint + +Linting is performed by prek, and the repo must define a prek.toml file at the +top level. The ``lint.yml`` workflow runs prek, and manages the installation +of other [FINISH ME - not clear how clang-format is fitting in here.]. +The exact definition of the ruff linting rules is defined in pyproject.toml. + +prek is intended to be run as a pre-commit hook. Run ``uv run prek install`` +on the first time you use it. If you hit an obscure problem with gitlab +URLs, run ``uv run prek install -c prek.toml``. + +If you are having problems with prek locally giving different results to +what's seen on CI, try running ``uv run prek cache clean``. + + +### Python C tests + +If the project defines a low-level Python C module, there is a special + +### C tests + +If the project has a C library, it will define some unit tests in C +using C Unit. These are built using meson and ninja and coverage +uploaded to CodeCov. From 4fcb799ef1ee7938978ada0591ec37f7c713f439 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 26 Feb 2026 15:43:13 +0000 Subject: [PATCH 072/106] Update --- repo_administration.md | 62 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/repo_administration.md b/repo_administration.md index 50138ad..38b7767 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -4,6 +4,68 @@ This document describes the standard layout and organisation principles for core tskit-dev projects. The GitHub workflows defined in this repo require that this layout is followed. + +## tskit-dev packages + +The tskit dev managed packages can be split into two groups: Python only, and Python and C. +The rules for administration are slightly different for the two. The repos are: + +**Python only** + +- tszip +- tstrait +- tsbrowse +- pyslim + +**Python and C** + +- tskit +- msprime +- kastore +- tsinfer + +## Releases + +### Python packages + +Releases are performed using the ``wheels.yml`` workflow in each repo using the +Trusted Publisher PyPI mechanism. Note that this mechanism requires that the +actual upload be performed from the source repository and not from a shared +workflow. There is therefore some duplication across the repos of this logic. + +To make a release first test that the process is working by pushing to +a branch named ``test-publish``: + +``` +git fetch upstream +git checkout upstream/main +git checkout -b test-publish +git push upstream test-publish +``` + +Then go to the Actions tab on github and watch the progress of the build. If +it all succeeds a release should be pushed to testPyPI (click on the +"Publish Distribition to TestPyPI" section to get the URL at the end). If this looks OK, then +proceed with the release on the GitHub UI. + +This action is only run on demand and not on push to main branch because +the binary wheel building is complex and expensive and there's little point in +running it over and over again. If it's broken when you get ready to do a +release, fix it. + +For Python-only repos the workflow is very simple and does nothing that the +python-packaging tests don't do. + +### Binary wheel building + +This is done using the shared workflowA + + +### C API releases + +DOCUMENT + + ## Dependency management We use uv exclusively for Python package management. All Python dependencies must From 83a5661fd390b466e409026749efeeb760f11e79 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 26 Feb 2026 15:45:27 +0000 Subject: [PATCH 073/106] Some details --- repo_administration.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/repo_administration.md b/repo_administration.md index 38b7767..b35fd67 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -58,7 +58,12 @@ python-packaging tests don't do. ### Binary wheel building -This is done using the shared workflowA +This is done using the shared workflow build-wheels.yml. This is using +cibuildwheel to do the heavy lifting. All configuration should be done +on the specific repo side using the pyproject.toml. One wrinkle to +note is that we specify that only builds native to the CPU architecture +occur on macOS because of the way that linking to GSL happens with +msprime. ### C API releases From 9b0dd70199f776213b0b150f9a8f8ab323841b41 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 26 Feb 2026 16:39:34 +0000 Subject: [PATCH 074/106] Update the docs by Claude --- repo_administration.md | 206 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 186 insertions(+), 20 deletions(-) diff --git a/repo_administration.md b/repo_administration.md index b35fd67..b92f518 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -14,6 +14,7 @@ The rules for administration are slightly different for the two. The repos are: - tszip - tstrait +- tsdate - tsbrowse - pyslim @@ -24,6 +25,70 @@ The rules for administration are slightly different for the two. The repos are: - kastore - tsinfer + +## Repository layout + +### Python-only repos + +The `pyproject.toml` lives at the repository root. Tests live in a `tests/` +directory at the root. The `prek.toml` lives at the root. + +### Python and C repos + +The Python package and its `pyproject.toml` live in a `python/` subdirectory. +The C library lives in a `c/` subdirectory. Tests are in `python/tests/`. +The `prek.toml` lives at the repo root (not in `python/`). All shared workflows +that accept a `pyproject-directory` input should be passed `python` for these +repos. + +Documentation (`docs/`) lives at the repository root and includes a +`doxygen/` subdirectory for generating C API docs. + + +## Dependency management + +We use uv exclusively for Python package management. All Python dependencies must +be declared explicitly in `pyproject.toml` using dependency groups. The dependencies +listed should be minimal, and pinned only where necessary. + +### Required dependency groups + +Every repo must define the following groups in `pyproject.toml`: + +- **test** – packages needed to run the test suite (pytest, pytest-cov, + pytest-xdist, and any domain-specific dependencies) +- **docs** – packages needed to build documentation (jupyter-book and + Sphinx extensions) +- **lint** – linting tools; must include pinned versions of `prek` and `ruff`. + Python+C repos must also include a pinned version of `clang-format`. +- **packaging** – `twine` and `validate-pyproject[all]` + +Python+C repos additionally require: + +- **wheels** – `cibuildwheel` (used by the build-wheels shared workflow) + +A **dev** group that includes all other groups is recommended for local +development convenience. + +### Version pinning in the lint group + +Lint tools must be pinned to exact versions in the `lint` group to ensure +deterministic results across local development and CI. Example: + +```toml +lint = [ + "clang-format==21.1.8", # Python+C repos only + "ruff==0.15.1", + "prek==0.3.3", +] +``` + +### Lock files + +All repos must maintain a `uv.lock` file at the root (or at the `pyproject.toml` +location for Python+C repos). This file is committed to the repository. + + ## Releases ### Python packages @@ -58,25 +123,38 @@ python-packaging tests don't do. ### Binary wheel building -This is done using the shared workflow build-wheels.yml. This is using +This is done using the shared workflow `build-wheels.yml`. This uses cibuildwheel to do the heavy lifting. All configuration should be done -on the specific repo side using the pyproject.toml. One wrinkle to -note is that we specify that only builds native to the CPU architecture -occur on macOS because of the way that linking to GSL happens with -msprime. +on the specific repo side using `pyproject.toml` under `[tool.cibuildwheel]`. + +The workflow builds an sdist first, then builds binary wheels from that sdist +across a matrix of operating systems: ubuntu-24.04, windows-2025, +macos-15-intel, and macos-14. On macOS, only wheels native to the runner's +CPU architecture are built (controlled by `CIBW_ARCHS_MACOS`); this is +required because of how linking to GSL works with msprime. +All cibuildwheel configuration belongs in `pyproject.toml`. This includes +specifying which Python versions to build for, architecture constraints, +system dependency installation (e.g. `before-all` for GSL), and +post-build wheel repair (e.g. `delvewheel` on Windows). ### C API releases -DOCUMENT +Repos that contain a C library (tskit, kastore) have a separate `release.yml` +workflow for publishing C API releases. The release is triggered by pushing a +tag whose name contains `C_` (e.g. `C_1.1.3`). Tags that do not contain `C_` +trigger a Python release instead. +The workflow uses `meson dist` to build a source tarball from the `c/` +directory and then creates a draft GitHub release with that tarball attached. +The draft must be manually reviewed and published. -## Dependency management +To make a C API release: + +1. Tag the commit with a name containing `C_`: `git tag C_X.Y.Z && git push upstream C_X.Y.Z` +2. Go to the Actions tab and verify the release workflow succeeds. +3. Go to the Releases page and publish the draft release. -We use uv exclusively for Python package management. All Python dependencies must -be declared explicitly in pyproject.toml using dependency groups, which map to tasks -like running python tests or building documentation. The dependencies listed should -be minimal, and pinned only where necessary. ## Test coverage @@ -89,23 +167,62 @@ and variables/Actions section (repository secrets). It is important to check tha uploads are working correctly - do this by examining the GitHub actions logs and looking at the output of the CodeCov upload command. +Coverage is uploaded with the following flags: + +- **python-tests** – from the standard Python test suite +- **c-python** – from the Python-C interface tests (Python+C repos only) +- **C** – from the C unit tests (Python+C repos only) + + ## Standard workflows ### Docs -All documentation is built using jupyterbook and is defined in the top-level "docs" +All documentation is built using jupyterbook and is defined in the top-level `docs/` directory. The ``docs.yml`` workflow builds documentation and incorporates it into the tskit.dev website. Two versions of the documentation are maintained, "latest" and "stable". The latest version is updated from the repo each time there is a merge. The stable version is updated on release. +The workflow requires: + +- A `docs` dependency group in `pyproject.toml` +- A `__PKG_VERSION__` placeholder string in `docs/_config.yml`, which is + replaced at build time with the package version derived from the installed package. + +For Python+C repos, pass `doxygen-directory: docs/doxygen` to generate C API +documentation with Doxygen before the JupyterBook build. + +On merge to `main`, the workflow triggers a rebuild of the tskit-site via +the AdminBot-tskit token. + ### Lint -Linting is performed by prek, and the repo must define a prek.toml file at the -top level. The ``lint.yml`` workflow runs prek, and manages the installation -of other [FINISH ME - not clear how clang-format is fitting in here.]. -The exact definition of the ruff linting rules is defined in pyproject.toml. +Linting is performed by prek using the ``lint.yml`` shared workflow. The workflow +installs only the `lint` dependency group (not the package itself, so no system +dependencies are required) and runs: + +``` +prek -c prek.toml run --all-files --show-diff-on-failure +``` + +The `prek.toml` file at the repo root defines all linting rules. The design +principle is to use only `builtin` and `local` hooks (no remote hook repos) to +ensure long-term stability and determinism. The standard hooks are: + +**Builtin hooks (all repos):** `check-added-large-files`, `check-merge-conflict`, +`mixed-line-ending`, `check-case-conflict`, `check-yaml`, `check-toml`. + +**Local hooks (all repos):** `ruff check --fix --force-exclude` and +`ruff format --force-exclude` applied to Python files. + +**Local hooks (Python+C repos only):** `clang-format -i` applied to C files. +`clang-format` is installed as part of the `lint` dependency group and invoked +as a system command. + +The `ruff` configuration (rules, target version, line length) is defined in +`pyproject.toml` under `[tool.ruff]`. prek is intended to be run as a pre-commit hook. Run ``uv run prek install`` on the first time you use it. If you hit an obscure problem with gitlab @@ -115,12 +232,61 @@ If you are having problems with prek locally giving different results to what's seen on CI, try running ``uv run prek cache clean``. +### Python packaging tests + +The ``python-packaging.yml`` shared workflow validates that the package can be +correctly built and distributed. It runs on ubuntu-24.04 and: + +1. Validates `pyproject.toml` using `validate-pyproject` +2. Builds sdist and wheel artifacts with `uv build` +3. Checks the artifacts with `twine check --strict` +4. Installs the built wheel into a fresh venv and optionally runs a CLI smoke test + +To run a CLI smoke test, pass the `cli-test-cmd` input (e.g. `tszip --help`). +This verifies the entry point is correctly declared. + + +### Python tests + +The ``python-tests.yml`` shared workflow runs the pytest suite on a configurable +Python version and OS. It installs only the `test` dependency group, runs pytest +with branch coverage and xdist parallelism (`-n2`), and uploads coverage to +CodeCov with the `python-tests` flag. + +Repos should test against a minimum and a recent Python version (currently 3.11 +and 3.13 or 3.14). Tests should run on ubuntu-24.04, macOS, and Windows. + +For Python+C repos, system dependencies (e.g. GSL) must be installed separately +in the calling workflow before invoking the shared workflow. + + ### Python C tests -If the project defines a low-level Python C module, there is a special +If the project defines a Python C extension module, use the ``python-c-tests.yml`` +shared workflow to run focused low-level tests against the C bindings. This +workflow runs on ubuntu-24.04 only and: + +1. Builds the C extension in-place with coverage instrumentation: + `CFLAGS=--coverage uv run python setup.py build_ext --inplace` +2. Runs the designated low-level test file (default: `tests/test_python_c.py`; + configurable via the `tests` input) +3. Generates a coverage report with gcovr and uploads it to CodeCov with the + `c-python` flag + +Pass `additional-apt-packages` for any system libraries required (e.g. `libgsl-dev`). + ### C tests -If the project has a C library, it will define some unit tests in C -using C Unit. These are built using meson and ninja and coverage -uploaded to CodeCov. +If the project has a C library, use the ``c-tests.yml`` shared workflow to run +the C unit tests. The workflow runs on ubuntu-24.04 and requires the `library-directory` +input pointing to the C library root (e.g. `c`). It: + +1. Installs system dependencies: `libcunit1-dev`, `libconfig-dev`, `ninja-build`, + `valgrind`, `clang`, plus any extras via `additional-apt-packages` +2. Installs meson and gcovr via uv tool +3. Builds with gcc and coverage enabled, runs tests, generates a coverage report, + and uploads it to CodeCov with the `C` flag +4. Builds again with clang and runs tests (no coverage upload) +5. Builds again with default settings and runs tests under valgrind with + `--leak-check=full` From 47bde26b015bec64693a5911a685c462c738ee1b Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 27 Feb 2026 13:19:13 +0000 Subject: [PATCH 075/106] Modularise the pre-build command --- .github/workflows/docs.yml | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6e78bdc..b7c2254 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,18 +16,19 @@ on: type: string default: '.' - doxygen-directory: - description: 'Directory to run doxygen from' + additional-apt-packages: + description: 'Additional APT packages to install' required: false type: string default: '' - additional-apt-packages: - description: 'Additional APT packages to install' + pre-build-command: + description: 'Command to run before building docs' required: false type: string default: '' + jobs: Docs: runs-on: ubuntu-24.04 @@ -45,9 +46,6 @@ jobs: with: python-version: '3.11' - # We could combine this with the doxygen command below, but in practise - # they are used mutually exclusively so there's nothing to be gained - # in making one apt install. - name: Install additional APT packages if: ${{ inputs.additional-apt-packages != '' }} run: sudo apt-get update && sudo apt-get install -y ${{ inputs.additional-apt-packages }} @@ -85,15 +83,11 @@ jobs: echo "PKG_VERSION=$PKG_VERSION" sed -i s/__PKG_VERSION__/${PKG_VERSION}/g docs/_config.yml - # Tskit has C code that we document using doxygen. This needs to be done - # before the jupyterbook build which incorporates it. - - name: Install doxygen (if required) - if: ${{ inputs.doxygen-directory != '' }} - run: sudo apt-get update && sudo apt-get install -y doxygen - - - name: Run doxygen (if required) - if: ${{ inputs.doxygen-directory != '' }} - run: cd ${{ inputs.doxygen-directory }} && doxygen + # We required doxygen and playwright commands before build on some + # repose. + - name: Run pre-build command + if: ${{ inputs.pre-build-command != '' }} + run: ${{ inputs.pre-build-command }} - name: Build documentation run: uv run --no-default-groups --project=${{ inputs.pyproject-directory }} jupyter-book build -nW docs/ From 615732cfa529cc5948183c6f87ad8a67e6f87c07 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 27 Feb 2026 15:23:45 +0000 Subject: [PATCH 076/106] Update docs section: local build instructions, CI requirements, remove doxygen-directory --- repo_administration.md | 46 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/repo_administration.md b/repo_administration.md index b92f518..e991305 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -184,17 +184,51 @@ into the tskit.dev website. Two versions of the documentation are maintained, "latest" and "stable". The latest version is updated from the repo each time there is a merge. The stable version is updated on release. -The workflow requires: +On merge to `main`, the workflow triggers a rebuild of the tskit-site via +the AdminBot-tskit token. + +#### Local building + +To build the documentation locally, run `make` from the `docs/` directory: + +``` +cd docs && make +``` + +This calls `docs/build.sh`, which runs: + +``` +uv run --project=.. --group docs jupyter-book build . -vnW --keep-going +``` + +(For Python+C repos, `--project=../python` is used instead of `--project=..`.) + +If the build fails, `build.sh` automatically prints any saved JupyterBook +error reports to help diagnose the failure. + +For Python+C repos the `Makefile` also runs Doxygen to generate C API XML +before calling `build.sh`. Doxygen is only re-run when the relevant C headers +have changed, so repeated `make` invocations are fast. + +To clean the build output: `make clean`. + +#### CI requirements + +The shared `docs.yml@main` workflow requires: - A `docs` dependency group in `pyproject.toml` - A `__PKG_VERSION__` placeholder string in `docs/_config.yml`, which is - replaced at build time with the package version derived from the installed package. + replaced at build time with the package version derived from the installed + package. -For Python+C repos, pass `doxygen-directory: docs/doxygen` to generate C API -documentation with Doxygen before the JupyterBook build. +For Python+C repos, Doxygen must be installed and run before the JupyterBook +build. Pass these inputs to the shared workflow: -On merge to `main`, the workflow triggers a rebuild of the tskit-site via -the AdminBot-tskit token. +```yaml +with: + additional-apt-packages: doxygen + pre-build-command: cd docs/doxygen && doxygen +``` ### Lint From 7e2864d6391c045b649a50d79900bb2e1f945492 Mon Sep 17 00:00:00 2001 From: Tskit Developers Date: Mon, 2 Mar 2026 12:21:49 +0000 Subject: [PATCH 077/106] Extract build logic into build-docs composite action Moves the shared docs build steps (apt install, uv sync, __PKG_VERSION__ substitution, pre-build command, cd docs && make) into a composite action at .github/actions/build-docs. The action operates on whatever is already checked out, so callers own the checkout step. The docs.yml reusable workflow is simplified to: checkout, call the composite action, trigger tskit-site rebuild on main. This also switches the build command from running jupyter-book directly to "cd docs && make", so the Makefile (and build.sh) is always the single entry point for docs builds. --- .github/actions/build-docs/action.yml | 73 +++++++++++++++++++++++++++ .github/workflows/docs.yml | 71 +++++--------------------- 2 files changed, 87 insertions(+), 57 deletions(-) create mode 100644 .github/actions/build-docs/action.yml diff --git a/.github/actions/build-docs/action.yml b/.github/actions/build-docs/action.yml new file mode 100644 index 0000000..0880003 --- /dev/null +++ b/.github/actions/build-docs/action.yml @@ -0,0 +1,73 @@ +# Composite action: build jupyterbook docs for a tskit-dev package. +# +# The repository must already be checked out before this action is called. +# It installs documentation dependencies, substitutes the __PKG_VERSION__ +# placeholder in docs/_config.yml, runs an optional pre-build command, then +# builds via "cd docs && make". +# +# Requirements on the target package: +# - pyproject.toml with a "docs" dependency group +# - docs/_config.yml containing a __PKG_VERSION__ placeholder +# - docs/Makefile with an "all" target that runs the jupyterbook build + +name: Build docs +description: Install deps and build jupyterbook docs for a tskit-dev package. + +inputs: + pyproject-directory: + description: 'Directory containing pyproject.toml (relative to repo root)' + required: false + default: '.' + + additional-apt-packages: + description: 'Space-separated list of additional APT packages to install' + required: false + default: '' + + pre-build-command: + description: 'Command to run after installing deps but before the docs build' + required: false + default: '' + +runs: + using: composite + steps: + - name: Install additional APT packages + if: inputs.additional-apt-packages != '' + shell: bash + run: sudo apt-get update && sudo apt-get install -y ${{ inputs.additional-apt-packages }} + + - name: Install uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.8.15" + + - name: Install doc deps + shell: bash + run: uv sync --no-default-groups --project=${{ inputs.pyproject-directory }} --locked --group docs + + - name: Set build version + shell: bash + run: | + set -euo pipefail + export PROJECT_DIR="${{ inputs.pyproject-directory }}" + PKG_VERSION="$( + uv run --no-default-groups --project "$PROJECT_DIR" python - < Date: Mon, 2 Mar 2026 12:23:24 +0000 Subject: [PATCH 078/106] Run jupyter-book directly rather than via Makefile --- .github/actions/build-docs/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build-docs/action.yml b/.github/actions/build-docs/action.yml index 0880003..3c256f0 100644 --- a/.github/actions/build-docs/action.yml +++ b/.github/actions/build-docs/action.yml @@ -70,4 +70,4 @@ runs: - name: Build documentation shell: bash - run: cd docs && make + run: uv run --no-default-groups --project=${{ inputs.pyproject-directory }} jupyter-book build -nW docs/ From 60e535ae399aa8fe234b9a422a0af089495738a7 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Tue, 3 Mar 2026 09:55:02 +0000 Subject: [PATCH 079/106] Rewrite repo_administration.md with principles-first layout Reorganise the document to lead with high-level principles and rationale, then cover layout, dependency management, CI workflows (including the new composite action), coverage, and releases. Keep content concise and avoid details that will quickly become stale. --- repo_administration.md | 363 +++++++++++++++-------------------------- 1 file changed, 134 insertions(+), 229 deletions(-) diff --git a/repo_administration.md b/repo_administration.md index e991305..9d55162 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -1,228 +1,146 @@ # Repo administration -This document describes the standard layout and organisation principles for -core tskit-dev projects. The GitHub workflows defined in this repo require -that this layout is followed. +## Overview +This document describes the standard layout, tooling, and CI conventions for +core tskit-dev Python packages. -## tskit-dev packages +**Core principles:** -The tskit dev managed packages can be split into two groups: Python only, and Python and C. -The rules for administration are slightly different for the two. The repos are: +- **Consistency.** All repos follow the same layout and conventions, making it + easy to work across the ecosystem and reducing the maintenance burden of each + individual repo. +- **Centralised CI logic.** Shared workflows and composite actions in this repo + (`tskit-dev/.github`) avoid duplication. Individual repos call into them via + `uses: tskit-dev/.github/.github/workflows/.yml@main` or + `uses: tskit-dev/.github/.github/actions/@main`. +- **Reproducibility.** All dependencies are managed exclusively with `uv` and + locked via committed `uv.lock` files. Lint tools are pinned to exact versions + so results are identical locally and in CI. +- **Minimal dependencies.** Dependencies should be declared in the appropriate + group and pinned only where necessary (lint tools being the main exception). -**Python only** +There are two categories of package in this ecosystem: -- tszip -- tstrait -- tsdate -- tsbrowse -- pyslim +- **Python-only:** tszip, tstrait, tsdate, tsbrowse, pyslim +- **Python+C:** tskit, msprime, kastore, tsinfer -**Python and C** - -- tskit -- msprime -- kastore -- tsinfer +The rules differ slightly between the two, as described below. ## Repository layout ### Python-only repos -The `pyproject.toml` lives at the repository root. Tests live in a `tests/` -directory at the root. The `prek.toml` lives at the root. +`pyproject.toml` and `uv.lock` live at the root. Tests live in `tests/`. +`prek.toml` lives at the root. -### Python and C repos +### Python+C repos The Python package and its `pyproject.toml` live in a `python/` subdirectory. -The C library lives in a `c/` subdirectory. Tests are in `python/tests/`. -The `prek.toml` lives at the repo root (not in `python/`). All shared workflows -that accept a `pyproject-directory` input should be passed `python` for these -repos. +The C library lives in `c/`. Tests are in `python/tests/`. `prek.toml` and +`uv.lock` live at the root. + +Documentation (`docs/`) lives at the repository root and includes a `doxygen/` +subdirectory for generating C API docs. -Documentation (`docs/`) lives at the repository root and includes a -`doxygen/` subdirectory for generating C API docs. +All shared workflows that accept a `pyproject-directory` input should be passed +`python` for these repos. ## Dependency management -We use uv exclusively for Python package management. All Python dependencies must -be declared explicitly in `pyproject.toml` using dependency groups. The dependencies -listed should be minimal, and pinned only where necessary. +We use `uv` exclusively for Python package management. -### Required dependency groups +### Required groups -Every repo must define the following groups in `pyproject.toml`: +Every repo must define these dependency groups in `pyproject.toml`: -- **test** – packages needed to run the test suite (pytest, pytest-cov, - pytest-xdist, and any domain-specific dependencies) -- **docs** – packages needed to build documentation (jupyter-book and - Sphinx extensions) -- **lint** – linting tools; must include pinned versions of `prek` and `ruff`. - Python+C repos must also include a pinned version of `clang-format`. +- **test** – pytest and test-only dependencies +- **docs** – JupyterBook and Sphinx extensions +- **lint** – linting tools (see below) - **packaging** – `twine` and `validate-pyproject[all]` Python+C repos additionally require: -- **wheels** – `cibuildwheel` (used by the build-wheels shared workflow) +- **wheels** – `cibuildwheel` -A **dev** group that includes all other groups is recommended for local -development convenience. +A **dev** group that includes all groups except `wheels` is recommended for +local development convenience. -### Version pinning in the lint group +### Lint pinning Lint tools must be pinned to exact versions in the `lint` group to ensure -deterministic results across local development and CI. Example: +deterministic results: ```toml +[dependency-groups] lint = [ - "clang-format==21.1.8", # Python+C repos only - "ruff==0.15.1", - "prek==0.3.3", + "clang-format==x.y.z", # Python+C repos only + "ruff==x.y.z", + "prek==x.y.z", ] ``` ### Lock files -All repos must maintain a `uv.lock` file at the root (or at the `pyproject.toml` -location for Python+C repos). This file is committed to the repository. - - -## Releases - -### Python packages - -Releases are performed using the ``wheels.yml`` workflow in each repo using the -Trusted Publisher PyPI mechanism. Note that this mechanism requires that the -actual upload be performed from the source repository and not from a shared -workflow. There is therefore some duplication across the repos of this logic. - -To make a release first test that the process is working by pushing to -a branch named ``test-publish``: - -``` -git fetch upstream -git checkout upstream/main -git checkout -b test-publish -git push upstream test-publish -``` - -Then go to the Actions tab on github and watch the progress of the build. If -it all succeeds a release should be pushed to testPyPI (click on the -"Publish Distribition to TestPyPI" section to get the URL at the end). If this looks OK, then -proceed with the release on the GitHub UI. - -This action is only run on demand and not on push to main branch because -the binary wheel building is complex and expensive and there's little point in -running it over and over again. If it's broken when you get ready to do a -release, fix it. - -For Python-only repos the workflow is very simple and does nothing that the -python-packaging tests don't do. - -### Binary wheel building - -This is done using the shared workflow `build-wheels.yml`. This uses -cibuildwheel to do the heavy lifting. All configuration should be done -on the specific repo side using `pyproject.toml` under `[tool.cibuildwheel]`. - -The workflow builds an sdist first, then builds binary wheels from that sdist -across a matrix of operating systems: ubuntu-24.04, windows-2025, -macos-15-intel, and macos-14. On macOS, only wheels native to the runner's -CPU architecture are built (controlled by `CIBW_ARCHS_MACOS`); this is -required because of how linking to GSL works with msprime. - -All cibuildwheel configuration belongs in `pyproject.toml`. This includes -specifying which Python versions to build for, architecture constraints, -system dependency installation (e.g. `before-all` for GSL), and -post-build wheel repair (e.g. `delvewheel` on Windows). - -### C API releases - -Repos that contain a C library (tskit, kastore) have a separate `release.yml` -workflow for publishing C API releases. The release is triggered by pushing a -tag whose name contains `C_` (e.g. `C_1.1.3`). Tags that do not contain `C_` -trigger a Python release instead. +A `uv.lock` file must be maintained and committed to the repository. Run +`uv lock` after any change to dependencies. For Python+C repos the lock file +lives alongside `pyproject.toml` in `python/`. -The workflow uses `meson dist` to build a source tarball from the `c/` -directory and then creates a draft GitHub release with that tarball attached. -The draft must be manually reviewed and published. -To make a C API release: +## CI workflows -1. Tag the commit with a name containing `C_`: `git tag C_X.Y.Z && git push upstream C_X.Y.Z` -2. Go to the Actions tab and verify the release workflow succeeds. -3. Go to the Releases page and publish the draft release. +Each repo calls shared workflows defined in this repository. The available +workflows are: +| Workflow | Purpose | +|---|---| +| `docs.yml` | Build and deploy documentation | +| `python-tests.yml` | Run the pytest suite with coverage | +| `lint.yml` | Run prek linting | +| `python-packaging.yml` | Validate packaging (sdist, wheel, twine) | +| `build-wheels.yml` | Build binary wheels (Python+C only) | +| `python-c-tests.yml` | Low-level C extension tests (Python+C only) | +| `c-tests.yml` | C unit tests with valgrind (Python+C only) | -## Test coverage - -Test coverage is monitored by CodeCov. When tests are run in CI, coverage is computed -and uploaded to CodeCov for each PR (and on merge). This is done using the -``CODECOV_TOKEN`` secret, which is generated by CodeCov and associated with a -given repo by GitHub. The token must be generated by logging in to the CodeCov UI, -copying the token and pasting it into the GitHub UI. This is in the settings/secrets -and variables/Actions section (repository secrets). It is important to check that -uploads are working correctly - do this by examining the GitHub actions logs and looking -at the output of the CodeCov upload command. - -Coverage is uploaded with the following flags: - -- **python-tests** – from the standard Python test suite -- **c-python** – from the Python-C interface tests (Python+C repos only) -- **C** – from the C unit tests (Python+C repos only) - - -## Standard workflows ### Docs -All documentation is built using jupyterbook and is defined in the top-level `docs/` -directory. The ``docs.yml`` workflow builds documentation and incorporates it -into the tskit.dev website. Two versions of the documentation are maintained, -"latest" and "stable". The latest version is updated from the repo each time -there is a merge. The stable version is updated on release. +Documentation is built with JupyterBook. Each repo has a `docs/` directory +containing a `Makefile`, `build.sh`, and JupyterBook configuration. -On merge to `main`, the workflow triggers a rebuild of the tskit-site via -the AdminBot-tskit token. - -#### Local building - -To build the documentation locally, run `make` from the `docs/` directory: +**Local build:** ``` cd docs && make ``` -This calls `docs/build.sh`, which runs: - -``` -uv run --project=.. --group docs jupyter-book build . -vnW --keep-going -``` +This calls `build.sh`, which runs JupyterBook with verbose output and error +reporting. For Python+C repos, the Makefile also runs Doxygen before the build +(only when C headers have changed, so repeated builds are fast). Clean build +output with `make clean`. -(For Python+C repos, `--project=../python` is used instead of `--project=..`.) +**CI build:** -If the build fails, `build.sh` automatically prints any saved JupyterBook -error reports to help diagnose the failure. +The `docs.yml` shared workflow calls the `build-docs` composite action +(`.github/actions/build-docs`), which: +1. Installs the `docs` dependency group +2. Substitutes the `__PKG_VERSION__` placeholder in `docs/_config.yml` with + the installed package version +3. Optionally runs a pre-build command (e.g. for Doxygen) +4. Runs JupyterBook -For Python+C repos the `Makefile` also runs Doxygen to generate C API XML -before calling `build.sh`. Doxygen is only re-run when the relevant C headers -have changed, so repeated `make` invocations are fast. +On merge to `main`, the workflow triggers a rebuild of the tskit.dev website. +Two documentation versions are maintained: **latest** (built from `main`) and +**stable** (built from the most recent release tag). -To clean the build output: `make clean`. - -#### CI requirements - -The shared `docs.yml@main` workflow requires: +**Requirements:** - A `docs` dependency group in `pyproject.toml` -- A `__PKG_VERSION__` placeholder string in `docs/_config.yml`, which is - replaced at build time with the package version derived from the installed - package. +- A `__PKG_VERSION__` placeholder string in `docs/_config.yml` -For Python+C repos, Doxygen must be installed and run before the JupyterBook -build. Pass these inputs to the shared workflow: +For Python+C repos, pass Doxygen setup as inputs to the shared workflow: ```yaml with: @@ -231,96 +149,83 @@ with: ``` -### Lint +### Tests -Linting is performed by prek using the ``lint.yml`` shared workflow. The workflow -installs only the `lint` dependency group (not the package itself, so no system -dependencies are required) and runs: - -``` -prek -c prek.toml run --all-files --show-diff-on-failure -``` +The `python-tests.yml` workflow runs pytest with branch coverage and parallel +execution, uploading results to CodeCov with the `python-tests` flag. Tests +should cover a minimum and a recent Python version across Linux, macOS, and +Windows. -The `prek.toml` file at the repo root defines all linting rules. The design -principle is to use only `builtin` and `local` hooks (no remote hook repos) to -ensure long-term stability and determinism. The standard hooks are: +For Python+C repos, required system libraries must be installed in the calling +workflow before invoking the shared workflow. -**Builtin hooks (all repos):** `check-added-large-files`, `check-merge-conflict`, -`mixed-line-ending`, `check-case-conflict`, `check-yaml`, `check-toml`. -**Local hooks (all repos):** `ruff check --fix --force-exclude` and -`ruff format --force-exclude` applied to Python files. +### Lint -**Local hooks (Python+C repos only):** `clang-format -i` applied to C files. -`clang-format` is installed as part of the `lint` dependency group and invoked -as a system command. +The `lint.yml` workflow installs only the `lint` group (no system dependencies +or package installation required) and runs prek against all files. The +`prek.toml` at the repo root defines all rules using only `builtin` and `local` +hooks for long-term stability. -The `ruff` configuration (rules, target version, line length) is defined in -`pyproject.toml` under `[tool.ruff]`. +Standard hooks cover: large file detection, merge conflict markers, line +endings, YAML/TOML validation, `ruff check`, and `ruff format`. Python+C repos +additionally run `clang-format` on C files. -prek is intended to be run as a pre-commit hook. Run ``uv run prek install`` -on the first time you use it. If you hit an obscure problem with gitlab -URLs, run ``uv run prek install -c prek.toml``. +Install prek as a pre-commit hook with `uv run prek install`. If local results +differ from CI, run `uv run prek cache clean`. -If you are having problems with prek locally giving different results to -what's seen on CI, try running ``uv run prek cache clean``. +### Packaging -### Python packaging tests +The `python-packaging.yml` workflow validates `pyproject.toml`, builds sdist +and wheel artifacts, and checks them with `twine --strict`. Pass a +`cli-test-cmd` input (e.g. `tszip --help`) for packages with entry points to +verify the entry point is correctly declared. -The ``python-packaging.yml`` shared workflow validates that the package can be -correctly built and distributed. It runs on ubuntu-24.04 and: -1. Validates `pyproject.toml` using `validate-pyproject` -2. Builds sdist and wheel artifacts with `uv build` -3. Checks the artifacts with `twine check --strict` -4. Installs the built wheel into a fresh venv and optionally runs a CLI smoke test +### Python+C specific -To run a CLI smoke test, pass the `cli-test-cmd` input (e.g. `tszip --help`). -This verifies the entry point is correctly declared. +**`python-c-tests.yml`** builds the C extension with coverage instrumentation +and runs focused low-level interface tests, uploading results to CodeCov with +the `c-python` flag. +**`c-tests.yml`** runs the C unit tests under gcc (with coverage), clang, and +valgrind. Coverage is uploaded with the `C` flag. -### Python tests -The ``python-tests.yml`` shared workflow runs the pytest suite on a configurable -Python version and OS. It installs only the `test` dependency group, runs pytest -with branch coverage and xdist parallelism (`-n2`), and uploads coverage to -CodeCov with the `python-tests` flag. +## Test coverage -Repos should test against a minimum and a recent Python version (currently 3.11 -and 3.13 or 3.14). Tests should run on ubuntu-24.04, macOS, and Windows. +Coverage is monitored by CodeCov. The `CODECOV_TOKEN` secret must be set in +each repo's GitHub Actions secrets. Verify uploads are working by checking the +Actions logs after a CI run. -For Python+C repos, system dependencies (e.g. GSL) must be installed separately -in the calling workflow before invoking the shared workflow. +Flags used: **python-tests**, **c-python** (Python+C only), **C** (Python+C +only). -### Python C tests +## Releases -If the project defines a Python C extension module, use the ``python-c-tests.yml`` -shared workflow to run focused low-level tests against the C bindings. This -workflow runs on ubuntu-24.04 only and: +### Python packages -1. Builds the C extension in-place with coverage instrumentation: - `CFLAGS=--coverage uv run python setup.py build_ext --inplace` -2. Runs the designated low-level test file (default: `tests/test_python_c.py`; - configurable via the `tests` input) -3. Generates a coverage report with gcovr and uploads it to CodeCov with the - `c-python` flag +Releases use the `wheels.yml` workflow via the Trusted Publisher PyPI +mechanism. Because Trusted Publisher requires the upload to originate from the +source repository, this workflow is not shared — each repo maintains its own +copy. -Pass `additional-apt-packages` for any system libraries required (e.g. `libgsl-dev`). +Test the release process first by pushing to a `test-publish` branch and +verifying that a test release appears on TestPyPI. Then trigger the actual +release via the GitHub UI. +### Binary wheels (Python+C only) -### C tests +The `build-wheels.yml` shared workflow uses `cibuildwheel` to build binary +wheels across Linux, macOS, and Windows. All configuration lives in +`pyproject.toml` under `[tool.cibuildwheel]`. -If the project has a C library, use the ``c-tests.yml`` shared workflow to run -the C unit tests. The workflow runs on ubuntu-24.04 and requires the `library-directory` -input pointing to the C library root (e.g. `c`). It: +### C API releases (Python+C only) -1. Installs system dependencies: `libcunit1-dev`, `libconfig-dev`, `ninja-build`, - `valgrind`, `clang`, plus any extras via `additional-apt-packages` -2. Installs meson and gcovr via uv tool -3. Builds with gcc and coverage enabled, runs tests, generates a coverage report, - and uploads it to CodeCov with the `C` flag -4. Builds again with clang and runs tests (no coverage upload) -5. Builds again with default settings and runs tests under valgrind with - `--leak-check=full` +Repos with a C library use a separate `release.yml` for C API releases. Push a +tag whose name contains `C_` (e.g. `C_1.1.3`) to trigger the workflow; tags +without `C_` trigger a Python release instead. The workflow builds a source +tarball with `meson dist` and creates a draft GitHub release for manual review +and publishing. From 0791991f27fc28a6ba3ba9a38a19b6581e1ece4f Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Tue, 3 Mar 2026 10:06:27 +0000 Subject: [PATCH 080/106] Distinguish external/internal C library repos; expand codecov.yml docs Split Python+C category into external C library (tskit, kastore) and internal C library (msprime, tsinfer) with their respective layout differences. Add a codecov.yml section covering carryforward flags, status check configuration, and ignored paths. --- repo_administration.md | 53 ++++++++++++++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/repo_administration.md b/repo_administration.md index 9d55162..043e4ec 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -20,12 +20,15 @@ core tskit-dev Python packages. - **Minimal dependencies.** Dependencies should be declared in the appropriate group and pinned only where necessary (lint tools being the main exception). -There are two categories of package in this ecosystem: +There are three categories of package in this ecosystem: - **Python-only:** tszip, tstrait, tsdate, tsbrowse, pyslim -- **Python+C:** tskit, msprime, kastore, tsinfer +- **Python+C (external C library):** tskit, kastore — the C library has a public API, + is independently versioned, and is consumed by other projects +- **Python+C (internal C library):** msprime, tsinfer — the C code is bundled as + part of the Python package and is not released separately -The rules differ slightly between the two, as described below. +The rules differ slightly between these categories, as described below. ## Repository layout @@ -35,15 +38,25 @@ The rules differ slightly between the two, as described below. `pyproject.toml` and `uv.lock` live at the root. Tests live in `tests/`. `prek.toml` lives at the root. -### Python+C repos +### Python+C repos (external C library) The Python package and its `pyproject.toml` live in a `python/` subdirectory. -The C library lives in `c/`. Tests are in `python/tests/`. `prek.toml` and -`uv.lock` live at the root. +The standalone C library lives in `c/`. Tests are in `python/tests/`. `prek.toml` +and `uv.lock` live at the root. Documentation (`docs/`) lives at the repository root and includes a `doxygen/` subdirectory for generating C API docs. +### Python+C repos (internal C library) + +The Python package and its `pyproject.toml` live in a `python/` subdirectory. +The C extension code lives in `lib/` within the `python/` subdirectory. Tests +are in `python/tests/`. `prek.toml` and `uv.lock` live at the root. + +There is no separate C API documentation or independent C release workflow. + +### All Python+C repos + All shared workflows that accept a `pyproject-directory` input should be passed `python` for these repos. @@ -202,6 +215,22 @@ Actions logs after a CI run. Flags used: **python-tests**, **c-python** (Python+C only), **C** (Python+C only). +### `codecov.yml` + +Each repo must include a `codecov.yml` at the repository root. This file +controls how CodeCov interprets and reports coverage data: + +- **Flag definitions.** Each flag (`python-tests`, `c-python`, `C`) is declared + with `carryforward: true` so that if a CI run only uploads a subset of flags + (e.g. on a docs-only change), CodeCov uses the most recent result for the + missing flags rather than treating them as zero. +- **Status checks.** Thresholds for the PR status check (patch and project + coverage) are configured here. Setting `informational: true` on checks that + are not yet stable avoids blocking merges while coverage is still being + established. +- **Ignored paths.** Test files and generated code can be excluded from + coverage totals so that the reported figure reflects production code only. + ## Releases @@ -222,10 +251,10 @@ The `build-wheels.yml` shared workflow uses `cibuildwheel` to build binary wheels across Linux, macOS, and Windows. All configuration lives in `pyproject.toml` under `[tool.cibuildwheel]`. -### C API releases (Python+C only) +### C API releases (external C library only) -Repos with a C library use a separate `release.yml` for C API releases. Push a -tag whose name contains `C_` (e.g. `C_1.1.3`) to trigger the workflow; tags -without `C_` trigger a Python release instead. The workflow builds a source -tarball with `meson dist` and creates a draft GitHub release for manual review -and publishing. +Repos with an external C library (tskit, kastore) use a separate `release.yml` +for C API releases. Push a tag whose name contains `C_` (e.g. `C_1.1.3`) to +trigger the workflow; tags without `C_` trigger a Python release instead. The +workflow builds a source tarball with `meson dist` and creates a draft GitHub +release for manual review and publishing. From 361a27fb95a42028991b051a527ad1169ec24d09 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Tue, 3 Mar 2026 10:11:41 +0000 Subject: [PATCH 081/106] Tweaks --- repo_administration.md | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/repo_administration.md b/repo_administration.md index 043e4ec..01d7d0d 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -22,7 +22,7 @@ core tskit-dev Python packages. There are three categories of package in this ecosystem: -- **Python-only:** tszip, tstrait, tsdate, tsbrowse, pyslim +- **Python-only:** tszip, tstrait, tsdate, tsbrowse, pyslim, tscompare, sc2ts - **Python+C (external C library):** tskit, kastore — the C library has a public API, is independently versioned, and is consumed by other projects - **Python+C (internal C library):** msprime, tsinfer — the C code is bundled as @@ -217,20 +217,8 @@ only). ### `codecov.yml` -Each repo must include a `codecov.yml` at the repository root. This file -controls how CodeCov interprets and reports coverage data: - -- **Flag definitions.** Each flag (`python-tests`, `c-python`, `C`) is declared - with `carryforward: true` so that if a CI run only uploads a subset of flags - (e.g. on a docs-only change), CodeCov uses the most recent result for the - missing flags rather than treating them as zero. -- **Status checks.** Thresholds for the PR status check (patch and project - coverage) are configured here. Setting `informational: true` on checks that - are not yet stable avoids blocking merges while coverage is still being - established. -- **Ignored paths.** Test files and generated code can be excluded from - coverage totals so that the reported figure reflects production code only. - +Each repo should include a `codecov.yml` at the repository root. This file +controls how CodeCov interprets and reports coverage data. ## Releases From a05fc3b7ac93ae97e03f4129f5b9bb3f5744a304 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Tue, 3 Mar 2026 15:52:50 +0000 Subject: [PATCH 082/106] Delete .github/workflows/docs-build-template.yml --- .github/workflows/docs-build-template.yml | 70 ----------------------- 1 file changed, 70 deletions(-) delete mode 100644 .github/workflows/docs-build-template.yml diff --git a/.github/workflows/docs-build-template.yml b/.github/workflows/docs-build-template.yml deleted file mode 100644 index 79df3d4..0000000 --- a/.github/workflows/docs-build-template.yml +++ /dev/null @@ -1,70 +0,0 @@ -# DEPRECATED - repos using this should move to the docs.yml -# workflow -name: Documentation Build Template - -on: - workflow_call: - inputs: - additional-setup: - description: 'Additional setup commands to run' - required: false - type: string - default: '' - make-command: - description: 'Make command for C modules' - required: false - type: string - default: '' - -jobs: - Docs: - runs-on: ubuntu-24.04 - steps: - - uses: styfle/cancel-workflow-action@0.12.1 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@v4.2.2 - with: - submodules: true - fetch-depth: 0 - - - uses: actions/setup-python@v5.4.0 - with: - python-version: '3.11' - - - name: Install uv - uses: astral-sh/setup-uv@v6 - with: - version: "0.8.15" - - - name: Install doc deps - run: | - - if [ -f pyproject.toml ]; then - uv pip install --system -r pyproject.toml --extra docs - elif [ -f python/pyproject.toml ]; then - uv pip install --system -r python/pyproject.toml --extra docs - else - echo "No pyproject.toml found in root or python/ directory" - exit 1 - fi - - - name: Additional setup - if: inputs.additional-setup != '' - run: ${{ inputs.additional-setup }} - - - name: Build C modules - if: inputs.make-command != '' - run: ${{ inputs.make-command }} - - - name: Build documentation - run: cd docs && make - - - name: Trigger tskit-site rebuild - if: github.ref == 'refs/heads/main' - run: | - curl -X POST https://api.github.com/repos/tskit-dev/tskit-site/dispatches \ - -H 'Accept: application/vnd.github.everest-preview+json' \ - -u AdminBot-tskit:${{ secrets.ADMINBOT_TOKEN }} \ - --data '{"event_type":"build-docs"}' From 8c9a0a9795bf1cc292dc05ae26ecd697c1756409 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Wed, 4 Mar 2026 10:38:48 +0000 Subject: [PATCH 083/106] Add install-slim option to docs --- .github/actions/build-docs/action.yml | 19 +++++++++++++++++++ .github/workflows/docs.yml | 6 ++++++ 2 files changed, 25 insertions(+) diff --git a/.github/actions/build-docs/action.yml b/.github/actions/build-docs/action.yml index 3c256f0..7d9a173 100644 --- a/.github/actions/build-docs/action.yml +++ b/.github/actions/build-docs/action.yml @@ -29,6 +29,11 @@ inputs: required: false default: '' + install-slim: + description: 'Whether to install SLiM' + required: false + default: false + runs: using: composite steps: @@ -37,6 +42,20 @@ runs: shell: bash run: sudo apt-get update && sudo apt-get install -y ${{ inputs.additional-apt-packages }} + - name: Set up micromamba + if: {{ inputs.install-slim }} + uses: mamba-org/setup-micromamba@v2.0.4 + + - name: Install SLiM + if: {{ inputs.install-slim }} + run: | + micromamba install -n base -y slim + echo "$MAMBA_ROOT_PREFIX/bin" >> $GITHUB_PATH + + - name: Verify SLiM + if: {{ inputs.install-slim }} + run: slim -v + - name: Install uv uses: astral-sh/setup-uv@v6 with: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 782e900..09b0977 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -29,6 +29,11 @@ on: type: string default: '' + install-slim: + description: 'Whether to install SLiM' + required: false + default: false + jobs: Docs: name: Build docs @@ -48,6 +53,7 @@ jobs: pyproject-directory: ${{ inputs.pyproject-directory }} additional-apt-packages: ${{ inputs.additional-apt-packages }} pre-build-command: ${{ inputs.pre-build-command }} + install-slim: ${{ inputs.install-slim }} - name: Trigger tskit-site rebuild if: github.ref == 'refs/heads/main' From 56c5fe874bb4bf98f7a8aa1923e38060e8802976 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Wed, 4 Mar 2026 10:46:18 +0000 Subject: [PATCH 084/106] Add missing type --- .github/workflows/docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 09b0977..540f40d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -32,6 +32,7 @@ on: install-slim: description: 'Whether to install SLiM' required: false + type: boolean default: false jobs: From c71802e9a08283297845d141d48d9416b1776ace Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Wed, 4 Mar 2026 10:48:15 +0000 Subject: [PATCH 085/106] Update action.yml --- .github/actions/build-docs/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/build-docs/action.yml b/.github/actions/build-docs/action.yml index 7d9a173..c43db40 100644 --- a/.github/actions/build-docs/action.yml +++ b/.github/actions/build-docs/action.yml @@ -3,12 +3,11 @@ # The repository must already be checked out before this action is called. # It installs documentation dependencies, substitutes the __PKG_VERSION__ # placeholder in docs/_config.yml, runs an optional pre-build command, then -# builds via "cd docs && make". +# builds the docs using jupyterbook # # Requirements on the target package: # - pyproject.toml with a "docs" dependency group # - docs/_config.yml containing a __PKG_VERSION__ placeholder -# - docs/Makefile with an "all" target that runs the jupyterbook build name: Build docs description: Install deps and build jupyterbook docs for a tskit-dev package. From c51249068aad5230efa3d869d6091af51631af68 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Wed, 4 Mar 2026 11:09:56 +0000 Subject: [PATCH 086/106] Explicit comparison with true --- .github/actions/build-docs/action.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/actions/build-docs/action.yml b/.github/actions/build-docs/action.yml index c43db40..684094d 100644 --- a/.github/actions/build-docs/action.yml +++ b/.github/actions/build-docs/action.yml @@ -31,7 +31,7 @@ inputs: install-slim: description: 'Whether to install SLiM' required: false - default: false + default: 'false' runs: using: composite @@ -42,17 +42,19 @@ runs: run: sudo apt-get update && sudo apt-get install -y ${{ inputs.additional-apt-packages }} - name: Set up micromamba - if: {{ inputs.install-slim }} + if: ${{ inputs.install-slim == 'true' }} uses: mamba-org/setup-micromamba@v2.0.4 - name: Install SLiM - if: {{ inputs.install-slim }} + if: ${{ inputs.install-slim == 'true' }} + shell: bash run: | micromamba install -n base -y slim echo "$MAMBA_ROOT_PREFIX/bin" >> $GITHUB_PATH - name: Verify SLiM - if: {{ inputs.install-slim }} + if: ${{ inputs.install-slim == 'true' }} + shell: bash run: slim -v - name: Install uv From b80d66fd7cd70e722a34559aad723b72f16e65ff Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Wed, 4 Mar 2026 13:36:29 +0000 Subject: [PATCH 087/106] Use build.sh to show build docs build failures --- .github/actions/build-docs/action.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actions/build-docs/action.yml b/.github/actions/build-docs/action.yml index 684094d..05db215 100644 --- a/.github/actions/build-docs/action.yml +++ b/.github/actions/build-docs/action.yml @@ -90,4 +90,5 @@ runs: - name: Build documentation shell: bash - run: uv run --no-default-groups --project=${{ inputs.pyproject-directory }} jupyter-book build -nW docs/ + working-directory: docs + run: ./build.sh From 8e7946520837047bc3dfe6560595cdb40d3476a1 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Wed, 4 Mar 2026 13:49:57 +0000 Subject: [PATCH 088/106] Switch to using Makefile for docs build --- .github/actions/build-docs/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build-docs/action.yml b/.github/actions/build-docs/action.yml index 05db215..b0788db 100644 --- a/.github/actions/build-docs/action.yml +++ b/.github/actions/build-docs/action.yml @@ -91,4 +91,4 @@ runs: - name: Build documentation shell: bash working-directory: docs - run: ./build.sh + run: make From 04bb39a989ceddac5320398d73f79444d3af5046 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 5 Mar 2026 11:02:50 +0000 Subject: [PATCH 089/106] Update repo_administration.md --- repo_administration.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/repo_administration.md b/repo_administration.md index 01d7d0d..d7e4fa3 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -95,12 +95,19 @@ lint = [ ] ``` +Ruff can be updated periodically, with any required new fixes or "ignores" +included. + ### Lock files A `uv.lock` file must be maintained and committed to the repository. Run `uv lock` after any change to dependencies. For Python+C repos the lock file lives alongside `pyproject.toml` in `python/`. +Lock files should be updated periodically (say every 6 months) as part of routine maintenance. +This provides the opportunity to detect and fix problems caused by upstream packages in +a controlled fashion. + ## CI workflows From a188f600860021324adce8ee4a5646c5ac65550e Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 6 Mar 2026 11:44:05 +0000 Subject: [PATCH 090/106] Update for C/kastore instructions --- repo_administration.md | 54 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/repo_administration.md b/repo_administration.md index d7e4fa3..110368b 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -246,10 +246,50 @@ The `build-wheels.yml` shared workflow uses `cibuildwheel` to build binary wheels across Linux, macOS, and Windows. All configuration lives in `pyproject.toml` under `[tool.cibuildwheel]`. -### C API releases (external C library only) - -Repos with an external C library (tskit, kastore) use a separate `release.yml` -for C API releases. Push a tag whose name contains `C_` (e.g. `C_1.1.3`) to -trigger the workflow; tags without `C_` trigger a Python release instead. The -workflow builds a source tarball with `meson dist` and creates a draft GitHub -release for manual review and publishing. +### tskit and kastore releases + +tskit and kastore each have an independently versioned C library and a Python package. +The steps below apply to both repos. + +#### C API release + +1. Prepare a PR that: + - Updates the version macros in the C header and `c/VERSION.txt` + - Updates `c/CHANGELOG.rst` with the release date and version, and checks for + completeness (comparing `git log --follow --oneline -- c` with + `git log --follow --oneline -- c/CHANGELOG.rst` may help) +2. Merge the PR. +3. Tag and push: + ```bash + git tag -a C_MAJOR.MINOR.PATCH -m "C API version C_MAJOR.MINOR.PATCH" + git push upstream --tags + ``` +4. After a couple of minutes, `release.yml` will create a draft release on the + repo's GitHub releases page. Copy the changelog into the release body and publish + (click the pencil icon). +5. After release: open a new section in `c/CHANGELOG.rst` for future development + and close the GitHub issue milestone. + +#### Python release + +1. Prepare a PR that: + - Sets the correct version in `python//_version.py` (PEP 440 format: + `MAJOR.MINOR.PATCH` for a normal release, `MAJOR.MINOR.PATCHbX` for a beta) + - Updates `python/CHANGELOG.rst` with the release date and version, and checks + for completeness (comparing `git log --follow --oneline -- python` with + `git log --follow --oneline -- python/CHANGELOG.rst` may help) +2. Merge the PR. +3. **Test on TestPyPI**: push to the `test-publish` branch. `wheels.yml` will build + wheels and publish them to TestPyPI. Verify the release looks correct. +4. Tag and push: + ```bash + git tag -a MAJOR.MINOR.PATCH -m "Python version MAJOR.MINOR.PATCH" + git push upstream --tags + ``` +5. `release.yml` will create a draft release on the releases page. Copy the changelog + into the release body and publish. +6. Publishing the release triggers `wheels.yml`, which builds binary wheels and uploads + them to production PyPI via Trusted Publisher. Check the Actions tab to confirm. +7. After release: open a new section in `python/CHANGELOG.rst`, bump + `python//_version.py` to `MAJOR.MINOR.PATCH.dev0`, and close the GitHub + issue milestone. From ac0423a7c1011d946e1959fc3c0f61873c264b4b Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 6 Mar 2026 12:39:49 +0000 Subject: [PATCH 091/106] Fix details for tskit releases --- repo_administration.md | 43 +++++++++++++++++------------------------- 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/repo_administration.md b/repo_administration.md index 110368b..dc1d133 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -104,8 +104,8 @@ A `uv.lock` file must be maintained and committed to the repository. Run `uv lock` after any change to dependencies. For Python+C repos the lock file lives alongside `pyproject.toml` in `python/`. -Lock files should be updated periodically (say every 6 months) as part of routine maintenance. -This provides the opportunity to detect and fix problems caused by upstream packages in +Lock files should be updated periodically (say every 6 months) as part of routine maintenance. +This provides the opportunity to detect and fix problems caused by upstream packages in a controlled fashion. @@ -261,35 +261,26 @@ The steps below apply to both repos. 2. Merge the PR. 3. Tag and push: ```bash + git fetch upstream + git checkout upstream/main git tag -a C_MAJOR.MINOR.PATCH -m "C API version C_MAJOR.MINOR.PATCH" - git push upstream --tags + git push upstream C_MAJOR.MINOR.PATCH ``` 4. After a couple of minutes, `release.yml` will create a draft release on the - repo's GitHub releases page. Copy the changelog into the release body and publish - (click the pencil icon). + repo's GitHub releases page. This workflow builds the release tarball and + makes it available in a draft release which you can then update via the + web UI in the Releases section. Update the release body with the changelog + contents and publish. 5. After release: open a new section in `c/CHANGELOG.rst` for future development and close the GitHub issue milestone. #### Python release -1. Prepare a PR that: - - Sets the correct version in `python//_version.py` (PEP 440 format: - `MAJOR.MINOR.PATCH` for a normal release, `MAJOR.MINOR.PATCHbX` for a beta) - - Updates `python/CHANGELOG.rst` with the release date and version, and checks - for completeness (comparing `git log --follow --oneline -- python` with - `git log --follow --oneline -- python/CHANGELOG.rst` may help) -2. Merge the PR. -3. **Test on TestPyPI**: push to the `test-publish` branch. `wheels.yml` will build - wheels and publish them to TestPyPI. Verify the release looks correct. -4. Tag and push: - ```bash - git tag -a MAJOR.MINOR.PATCH -m "Python version MAJOR.MINOR.PATCH" - git push upstream --tags - ``` -5. `release.yml` will create a draft release on the releases page. Copy the changelog - into the release body and publish. -6. Publishing the release triggers `wheels.yml`, which builds binary wheels and uploads - them to production PyPI via Trusted Publisher. Check the Actions tab to confirm. -7. After release: open a new section in `python/CHANGELOG.rst`, bump - `python//_version.py` to `MAJOR.MINOR.PATCH.dev0`, and close the GitHub - issue milestone. +This follows the standard Python release process for binary wheels *except* +we must manually set the version number. So, follow all the steps for a +standard Python release except: + +- (-1) Set the version number in `python//_version.py` +- (n + ) Post release, set the version number to the next version ".dev0", + i.e., if we've just tagged 1.2.1, we set the development version to + 1.2.2.dev0. From 57aa255fca48b1274aadbf5ab9a1bd66d7bea3bc Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 6 Mar 2026 12:55:49 +0000 Subject: [PATCH 092/106] Details for tskit and general python release process. --- repo_administration.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/repo_administration.md b/repo_administration.md index dc1d133..3b83893 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -246,6 +246,37 @@ The `build-wheels.yml` shared workflow uses `cibuildwheel` to build binary wheels across Linux, macOS, and Windows. All configuration lives in `pyproject.toml` under `[tool.cibuildwheel]`. +### Standard python release process + +1. Create a PR updating the CHANGELOG with the correct version number + (and updating the version manually for repos that don't use `setuptools_scm`; + see specific instructions for kastore/tskit below). Merge this PR. +2. Create a branch called `test-publish` on upstream (either manually from the + command line or by using the GitHub UI). This will trigger the wheels.yml + GitHub action that builds the Python release artifacts and publishes them + using PyPI Trusted publishing. When we push to `test-publish` which pushes + the artifacts to Test PyPI, and when we create a full release it pushes them + to PyPI. Once the branch has been created, go to the Actions section on + Github and watch for the "Publish Python release" workflow associated with + "test-publish". This should succeed. It's important to perform this step + for repos that build binary releases (msprime, tskit, tsinfer, kastore) as + this wheel-building step **is not tested** between releases. If it fails, + this is the time to fix it before the release is actually made. +3. Once a distribution has been successfully uploaded to TestPyPI you can + delete the `test-publish` branch (using the UI) and create a release. + This is done by using the "Releases" section on GitHub. Click on "Draft + new release". In the "tag" box enter the version number of the new release + and click on "create new tag". This will create the new tag when the release + is published. Fill in the release body with the CHANGELOG contents + (the "latest" version of the docs on tskit.dev will have most of the formatting + done already and is a handy place to copy from). Click on "publish release". + Check the "Publish Python release" action again to make sure that the release + succeeds and verify on PyPI. +4. Once the release has been published, open a PR with any post-release tasks. + Usually this is just opening a new section in the CHANGELOG, but repos that + do not use `setuptools_scm` will also require the version number to be updated + manually also. + ### tskit and kastore releases tskit and kastore each have an independently versioned C library and a Python package. From 7d76f63d4687b49a213f95a3d40c49304d1766a8 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 6 Mar 2026 13:28:19 +0000 Subject: [PATCH 093/106] Final details on releases. --- repo_administration.md | 114 +++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 66 deletions(-) diff --git a/repo_administration.md b/repo_administration.md index 3b83893..d3eab69 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -54,6 +54,8 @@ The C extension code lives in `lib/` within the `python/` subdirectory. Tests are in `python/tests/`. `prek.toml` and `uv.lock` live at the root. There is no separate C API documentation or independent C release workflow. +These repos (msprime, tsinfer) use `setuptools_scm` for versioning, so no manual +version file needs to be maintained. ### All Python+C repos @@ -229,66 +231,50 @@ controls how CodeCov interprets and reports coverage data. ## Releases -### Python packages - -Releases use the `wheels.yml` workflow via the Trusted Publisher PyPI -mechanism. Because Trusted Publisher requires the upload to originate from the -source repository, this workflow is not shared — each repo maintains its own -copy. - -Test the release process first by pushing to a `test-publish` branch and -verifying that a test release appears on TestPyPI. Then trigger the actual -release via the GitHub UI. - -### Binary wheels (Python+C only) - -The `build-wheels.yml` shared workflow uses `cibuildwheel` to build binary -wheels across Linux, macOS, and Windows. All configuration lives in -`pyproject.toml` under `[tool.cibuildwheel]`. - -### Standard python release process - -1. Create a PR updating the CHANGELOG with the correct version number - (and updating the version manually for repos that don't use `setuptools_scm`; - see specific instructions for kastore/tskit below). Merge this PR. -2. Create a branch called `test-publish` on upstream (either manually from the - command line or by using the GitHub UI). This will trigger the wheels.yml - GitHub action that builds the Python release artifacts and publishes them - using PyPI Trusted publishing. When we push to `test-publish` which pushes - the artifacts to Test PyPI, and when we create a full release it pushes them - to PyPI. Once the branch has been created, go to the Actions section on - Github and watch for the "Publish Python release" workflow associated with - "test-publish". This should succeed. It's important to perform this step - for repos that build binary releases (msprime, tskit, tsinfer, kastore) as - this wheel-building step **is not tested** between releases. If it fails, - this is the time to fix it before the release is actually made. -3. Once a distribution has been successfully uploaded to TestPyPI you can - delete the `test-publish` branch (using the UI) and create a release. - This is done by using the "Releases" section on GitHub. Click on "Draft - new release". In the "tag" box enter the version number of the new release - and click on "create new tag". This will create the new tag when the release - is published. Fill in the release body with the CHANGELOG contents - (the "latest" version of the docs on tskit.dev will have most of the formatting - done already and is a handy place to copy from). Click on "publish release". - Check the "Publish Python release" action again to make sure that the release - succeeds and verify on PyPI. -4. Once the release has been published, open a PR with any post-release tasks. - Usually this is just opening a new section in the CHANGELOG, but repos that - do not use `setuptools_scm` will also require the version number to be updated - manually also. +Python releases use the `wheels.yml` workflow via the Trusted Publisher PyPI mechanism. +Because Trusted Publisher requires the upload to originate from the source repository, +this workflow is not shared — each repo maintains its own copy. + +Python+C repos additionally use the shared `build-wheels.yml` workflow, which uses +`cibuildwheel` to build binary wheels across Linux, macOS, and Windows. All +configuration lives in `pyproject.toml` under `[tool.cibuildwheel]`. + +tskit and kastore also have a repo-specific `release-c.yml` workflow solely for C API +releases (see below). This is separate from the Python release process. + +### Standard Python release process + +> **tskit and kastore:** follow the [tskit and kastore releases](#tskit-and-kastore-releases) +> section below instead of this standard process. + +1. Prepare a PR that updates the CHANGELOG with the correct version number. Merge this PR. +2. Push to the `test-publish` branch on upstream to trigger `wheels.yml`, which builds + release artifacts and publishes them to TestPyPI. Check the "Publish Python release" + action succeeds. This step is especially important for repos with binary wheels + (msprime, tsinfer, tskit, kastore) as the wheel-building step is **not tested** between releases. +3. Once the TestPyPI upload succeeds, delete the `test-publish` branch. Go to the + "Releases" section on GitHub and click "Draft new release". Enter the version number + in the tag box and click "create new tag" (the tag is created when the release is + published). Fill in the release body with the CHANGELOG contents (the "latest" docs + on tskit.dev have most of the formatting done already). Click "Publish release" and + confirm the "Publish Python release" action succeeds on PyPI. +4. Open a post-release PR that opens a new section in the CHANGELOG. ### tskit and kastore releases -tskit and kastore each have an independently versioned C library and a Python package. -The steps below apply to both repos. +tskit and kastore each have an independently versioned C library and a Python package, +and do not use `setuptools_scm`. The steps below apply to both repos. #### C API release +`release-c.yml` handles C API releases only — it is triggered by tags containing `C_` +and has no effect on the Python package. + 1. Prepare a PR that: - Updates the version macros in the C header and `c/VERSION.txt` - - Updates `c/CHANGELOG.rst` with the release date and version, and checks for - completeness (comparing `git log --follow --oneline -- c` with - `git log --follow --oneline -- c/CHANGELOG.rst` may help) + - Updates `c/CHANGELOG.rst` with the release date and version; check completeness + by comparing `git log --follow --oneline -- c` with + `git log --follow --oneline -- c/CHANGELOG.rst` 2. Merge the PR. 3. Tag and push: ```bash @@ -297,21 +283,17 @@ The steps below apply to both repos. git tag -a C_MAJOR.MINOR.PATCH -m "C API version C_MAJOR.MINOR.PATCH" git push upstream C_MAJOR.MINOR.PATCH ``` -4. After a couple of minutes, `release.yml` will create a draft release on the - repo's GitHub releases page. This workflow builds the release tarball and - makes it available in a draft release which you can then update via the - web UI in the Releases section. Update the release body with the changelog - contents and publish. -5. After release: open a new section in `c/CHANGELOG.rst` for future development - and close the GitHub issue milestone. +4. After a couple of minutes, `release-c.yml` will build the release tarball and create + a draft release on the repo's GitHub releases page. Update the release body with the + changelog contents and publish. +5. Open a post-release PR that opens a new section in `c/CHANGELOG.rst` and closes the + GitHub issue milestone. #### Python release -This follows the standard Python release process for binary wheels *except* -we must manually set the version number. So, follow all the steps for a -standard Python release except: +Follow the standard Python release process above, with two differences: -- (-1) Set the version number in `python//_version.py` -- (n + ) Post release, set the version number to the next version ".dev0", - i.e., if we've just tagged 1.2.1, we set the development version to - 1.2.2.dev0. +- In step 1, also set the version in `python//_version.py` (these repos + do not use `setuptools_scm`). +- In step 4, also bump the version in `python//_version.py` to + `MAJOR.MINOR.PATCH.dev1`. From a62f7527c8aa331fffa54b5cddac32e00cc60373 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 6 Mar 2026 13:42:06 +0000 Subject: [PATCH 094/106] Details on the website --- repo_administration.md | 95 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/repo_administration.md b/repo_administration.md index d3eab69..9562673 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -215,6 +215,101 @@ the `c-python` flag. valgrind. Coverage is uploaded with the `C` flag. +## The tskit.dev website + +The [tskit-site](https://github.com/tskit-dev/tskit-site) repository is the source for +the [tskit.dev](https://tskit.dev) website, including the landing page and the hosted +documentation for all packages in the ecosystem. It is a Jekyll site deployed to GitHub +Pages via the `gh-pages` branch. + +### Site build overview + +The `deploy.yml` workflow orchestrates the full build. It runs on: + +- Every push to `main` (live deployment) +- Every pull request (preview artifact only — not deployed) +- A daily scheduled cron job (keeps the site up to date with any upstream changes) +- A `repository_dispatch` event (triggered automatically by per-repo doc builds — see below) +- Manual `workflow_dispatch` + +`deploy.yml` calls three sub-workflows in parallel, then assembles and deploys the result: + +- **`build-core.yml`** — builds the Jekyll site itself (landing page, software listings, + news, etc.) using Ruby/Grunt +- **`build-docs.yml`** — builds per-package documentation for both `latest` (from `main`) + and `stable` (from the most recent release tag) versions +- **`build-extras.yml`** — imports the tutorials site (from `tskit-dev/tutorials`), rust + tutorials (from `tskit-dev/tskit-rust`), and builds the tskit-explore JupyterLite app + +The assembled site is deployed to `gh-pages`. Package docs land at +`//docs/latest` and `//docs/stable`; a redirect page at +`//docs/index.html` points visitors to the `stable` version by default. + +### How per-repo doc builds integrate + +Each repo's own `docs.yml` uses the shared `docs.yml` workflow defined in this +repository. On merge to `main`, after a successful build, the shared workflow fires a +`repository_dispatch` event to `tskit-site`, which triggers a full site rebuild. This +means that merging a documentation change to any repo automatically propagates to +tskit.dev within a few minutes. + +The `build-docs.yml` in tskit-site uses the same shared `build-docs` composite action +(`.github/actions/build-docs`) that the per-repo workflow uses, and passes the same +inputs (`pyproject-directory`, `additional-apt-packages`, `pre-build-command`). Builds +are cached by commit SHA, so a package's docs are only rebuilt when its `main` branch +has a new commit. + +### What changing the local docs build means + +When making changes that affect how docs are built (e.g. new APT dependencies, a new +pre-build command, changes to `build.sh` or the `docs` dependency group), be aware that +the build runs in two places: + +1. **The repo's own `docs.yml`** — configured via inputs to the shared `docs.yml` + workflow in the repo's `.github/workflows/docs.yml`. +2. **tskit-site's `build-docs.yml`** — has a matrix entry per package that passes the + same inputs. This file must be updated in the `tskit-dev/tskit-site` repo to match. + +If these two are out of sync, the per-repo CI will pass but the tskit-site build will +fail or produce incorrect output. The relevant matrix entry in `build-docs.yml` is +identified by the package name and specifies `additional-apt-packages`, +`pre-build-command`, `pyproject-directory`, and a `cache-version` (bump this whenever +the build environment changes to force a cache miss). + +Changes to `build.sh`, `docs/_config.yml`, or the `docs` dependency group are picked up +automatically on the next run with no changes needed to tskit-site. + +### Stable and latest versions + +`latest` is built from `main` on every new commit. `stable` is built from the most +recent release tag, determined by sorting all tags and selecting the newest that does +not contain `b`, `a`, or `C_` in its name (i.e. no pre-releases or C API tags). + +The critical distinction: **stable docs are built from the repo code at the tag, but +using the current shared build infrastructure** — the pinned `build-docs` action version +and whatever `additional-apt-packages` and `pre-build-command` are currently in the +tskit-site matrix entry for that package. The stable build does not use `main`'s +`build.sh` or `docs/` content; it uses whatever was committed at the tag. + +This means: + +- **Content and structural changes** (new pages, edited text, updated notebooks) only + affect `latest` until a new release is tagged. `stable` continues to serve the old + content. This is expected behaviour. + +- **Build environment changes** (new APT packages, changed `pre-build-command`) require + updating the tskit-site matrix entry, which applies to **both** `latest` and `stable`. + If the stable release's code is not compatible with the updated build environment, + the stable build will break. + +- **Bumping `cache-version`** in the tskit-site matrix entry forces a cache miss for + both versions. This is required when the build environment changes, but it also means + the stable docs will be fully rebuilt — so the same compatibility concern applies. + +When making a change that alters the build environment, verify that the stable version +still builds correctly, or make a new release first so that `stable` and `latest` point +to the same code. + ## Test coverage Coverage is monitored by CodeCov. The `CODECOV_TOKEN` secret must be set in From 718d5d54281641142275fd01164a828bb93b703f Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 6 Mar 2026 13:55:46 +0000 Subject: [PATCH 095/106] Minor rearrangement --- repo_administration.md | 181 +++++++++++++++++++++-------------------- 1 file changed, 94 insertions(+), 87 deletions(-) diff --git a/repo_administration.md b/repo_administration.md index 9562673..1122489 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -54,8 +54,6 @@ The C extension code lives in `lib/` within the `python/` subdirectory. Tests are in `python/tests/`. `prek.toml` and `uv.lock` live at the root. There is no separate C API documentation or independent C release workflow. -These repos (msprime, tsinfer) use `setuptools_scm` for versioning, so no manual -version file needs to be maintained. ### All Python+C repos @@ -178,8 +176,9 @@ execution, uploading results to CodeCov with the `python-tests` flag. Tests should cover a minimum and a recent Python version across Linux, macOS, and Windows. -For Python+C repos, required system libraries must be installed in the calling -workflow before invoking the shared workflow. +Some repos have their own version of the tests.yml and don't use the shared +workflow because bespoke actions need to be taken or difficult system libraries +need to be installed. ### Lint @@ -215,6 +214,97 @@ the `c-python` flag. valgrind. Coverage is uploaded with the `C` flag. +## Test coverage + +Coverage is monitored by CodeCov. The `CODECOV_TOKEN` secret must be set in +each repo's GitHub Actions secrets. Verify uploads are working by checking the +Actions logs after a CI run. + +Flags used: **python-tests**, **c-python** (Python+C only), **C** (Python+C +only). + +### `codecov.yml` + +Each repo should include a `codecov.yml` at the repository root. This file +controls how CodeCov interprets and reports coverage data. + +## Releases + +Python releases use the `wheels.yml` workflow via the Trusted Publisher PyPI mechanism. +Because Trusted Publisher requires the upload to originate from the source repository, +this workflow is not shared — each repo maintains its own copy. + +Python+C repos additionally use the shared `build-wheels.yml` workflow, which uses +`cibuildwheel` to build binary wheels across Linux, macOS, and Windows. All +configuration lives in `pyproject.toml` under `[tool.cibuildwheel]`. + +tskit and kastore also have a repo-specific `release-c.yml` workflow solely for C API +releases (see below). This is separate from the Python release process. + + +### Version management + +Most repos use `setuptools_scm` for versioning, which means that versions are automatically +created from git tags. This approach doesn't work for kastore and tskit, though, and so they +maintain version numbers manually. + +### Standard Python release process + +> **tskit and kastore:** follow the [tskit and kastore releases](#tskit-and-kastore-releases) +> section below instead of this standard process. + +1. Prepare a PR that updates the CHANGELOG with the correct version number. Merge this PR. +2. Push to the `test-publish` branch on upstream to trigger `wheels.yml`, which builds + release artifacts and publishes them to TestPyPI. Check the "Publish Python release" + action succeeds. This step is especially important for repos with binary wheels + (msprime, tsinfer, tskit, kastore) as the wheel-building step is **not tested** between releases. +3. Once the TestPyPI upload succeeds, delete the `test-publish` branch. Go to the + "Releases" section on GitHub and click "Draft new release". Enter the version number + in the tag box and click "create new tag" (the tag is created when the release is + published). Fill in the release body with the CHANGELOG contents (the "latest" docs + on tskit.dev have most of the formatting done already). Click "Publish release" and + confirm the "Publish Python release" action succeeds on PyPI. +4. Open a post-release PR that opens a new section in the CHANGELOG. + +### tskit and kastore releases + +tskit and kastore each have an independently versioned C library and a Python package, +and do not use `setuptools_scm`. The steps below apply to both repos. + +#### C API release + +`release-c.yml` handles C API releases only — it is triggered by tags containing `C_` +and has no effect on the Python package. + +1. Prepare a PR that: + - Updates the version macros in the C header and `c/VERSION.txt` + - Updates `c/CHANGELOG.rst` with the release date and version; check completeness + by comparing `git log --follow --oneline -- c` with + `git log --follow --oneline -- c/CHANGELOG.rst` +2. Merge the PR. +3. Tag and push: + ```bash + git fetch upstream + git checkout upstream/main + git tag -a C_MAJOR.MINOR.PATCH -m "C API version C_MAJOR.MINOR.PATCH" + git push upstream C_MAJOR.MINOR.PATCH + ``` +4. After a couple of minutes, `release-c.yml` will build the release tarball and create + a draft release on the repo's GitHub releases page. Update the release body with the + changelog contents and publish. +5. Open a post-release PR that opens a new section in `c/CHANGELOG.rst` and closes the + GitHub issue milestone. + +#### Python release + +Follow the standard Python release process above, with two differences: + +- In step 1, also set the version in `python//_version.py` (these repos + do not use `setuptools_scm`). +- In step 4, also bump the version in `python//_version.py` to + `MAJOR.MINOR.PATCH.dev1`. + + ## The tskit.dev website The [tskit-site](https://github.com/tskit-dev/tskit-site) repository is the source for @@ -309,86 +399,3 @@ This means: When making a change that alters the build environment, verify that the stable version still builds correctly, or make a new release first so that `stable` and `latest` point to the same code. - -## Test coverage - -Coverage is monitored by CodeCov. The `CODECOV_TOKEN` secret must be set in -each repo's GitHub Actions secrets. Verify uploads are working by checking the -Actions logs after a CI run. - -Flags used: **python-tests**, **c-python** (Python+C only), **C** (Python+C -only). - -### `codecov.yml` - -Each repo should include a `codecov.yml` at the repository root. This file -controls how CodeCov interprets and reports coverage data. - -## Releases - -Python releases use the `wheels.yml` workflow via the Trusted Publisher PyPI mechanism. -Because Trusted Publisher requires the upload to originate from the source repository, -this workflow is not shared — each repo maintains its own copy. - -Python+C repos additionally use the shared `build-wheels.yml` workflow, which uses -`cibuildwheel` to build binary wheels across Linux, macOS, and Windows. All -configuration lives in `pyproject.toml` under `[tool.cibuildwheel]`. - -tskit and kastore also have a repo-specific `release-c.yml` workflow solely for C API -releases (see below). This is separate from the Python release process. - -### Standard Python release process - -> **tskit and kastore:** follow the [tskit and kastore releases](#tskit-and-kastore-releases) -> section below instead of this standard process. - -1. Prepare a PR that updates the CHANGELOG with the correct version number. Merge this PR. -2. Push to the `test-publish` branch on upstream to trigger `wheels.yml`, which builds - release artifacts and publishes them to TestPyPI. Check the "Publish Python release" - action succeeds. This step is especially important for repos with binary wheels - (msprime, tsinfer, tskit, kastore) as the wheel-building step is **not tested** between releases. -3. Once the TestPyPI upload succeeds, delete the `test-publish` branch. Go to the - "Releases" section on GitHub and click "Draft new release". Enter the version number - in the tag box and click "create new tag" (the tag is created when the release is - published). Fill in the release body with the CHANGELOG contents (the "latest" docs - on tskit.dev have most of the formatting done already). Click "Publish release" and - confirm the "Publish Python release" action succeeds on PyPI. -4. Open a post-release PR that opens a new section in the CHANGELOG. - -### tskit and kastore releases - -tskit and kastore each have an independently versioned C library and a Python package, -and do not use `setuptools_scm`. The steps below apply to both repos. - -#### C API release - -`release-c.yml` handles C API releases only — it is triggered by tags containing `C_` -and has no effect on the Python package. - -1. Prepare a PR that: - - Updates the version macros in the C header and `c/VERSION.txt` - - Updates `c/CHANGELOG.rst` with the release date and version; check completeness - by comparing `git log --follow --oneline -- c` with - `git log --follow --oneline -- c/CHANGELOG.rst` -2. Merge the PR. -3. Tag and push: - ```bash - git fetch upstream - git checkout upstream/main - git tag -a C_MAJOR.MINOR.PATCH -m "C API version C_MAJOR.MINOR.PATCH" - git push upstream C_MAJOR.MINOR.PATCH - ``` -4. After a couple of minutes, `release-c.yml` will build the release tarball and create - a draft release on the repo's GitHub releases page. Update the release body with the - changelog contents and publish. -5. Open a post-release PR that opens a new section in `c/CHANGELOG.rst` and closes the - GitHub issue milestone. - -#### Python release - -Follow the standard Python release process above, with two differences: - -- In step 1, also set the version in `python//_version.py` (these repos - do not use `setuptools_scm`). -- In step 4, also bump the version in `python//_version.py` to - `MAJOR.MINOR.PATCH.dev1`. From 02fb2dcb4fe4781ac9fc46c397e715654bb7f52a Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 6 Mar 2026 13:56:53 +0000 Subject: [PATCH 096/106] Add toc --- repo_administration.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/repo_administration.md b/repo_administration.md index 1122489..6462ecc 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -1,5 +1,17 @@ # Repo administration +## Contents + +- [Overview](#overview) +- [Repository layout](#repository-layout) +- [Dependency management](#dependency-management) +- [CI workflows](#ci-workflows) +- [Test coverage](#test-coverage) +- [Releases](#releases) + - [Standard Python release process](#standard-python-release-process) + - [tskit and kastore releases](#tskit-and-kastore-releases) +- [The tskit.dev website](#the-tskitdev-website) + ## Overview This document describes the standard layout, tooling, and CI conventions for From ef7faea4cef5a7c3c368efe5671ca3853c0ef6d8 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 6 Mar 2026 14:21:07 +0000 Subject: [PATCH 097/106] Add Python version updates info --- repo_administration.md | 46 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/repo_administration.md b/repo_administration.md index 6462ecc..8640c9c 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -7,6 +7,7 @@ - [Dependency management](#dependency-management) - [CI workflows](#ci-workflows) - [Test coverage](#test-coverage) +- [Python version support](#python-version-support) - [Releases](#releases) - [Standard Python release process](#standard-python-release-process) - [tskit and kastore releases](#tskit-and-kastore-releases) @@ -240,6 +241,51 @@ only). Each repo should include a `codecov.yml` at the repository root. This file controls how CodeCov interprets and reports coverage data. +## Python version support + +When adding or removing support for a Python version, update the following in every +affected repo: + +### All repos + +**`pyproject.toml`** (or `python/pyproject.toml` for Python+C repos): + +- `requires-python` — set the floor version, e.g. `">=3.12"`. +- `classifiers` — add or remove the corresponding + `"Programming Language :: Python :: 3.X"` entry. This is informational (displayed + on PyPI) but should be kept in sync with `requires-python` and the test matrix. +- `[tool.ruff] target-version` — must match the minimum supported version (e.g. + `"py312"`). Forgetting this means ruff silently permits syntax that is invalid on + the stated floor version. + +**`.github/workflows/tests.yml`**: + +- Update the `matrix.python` list. Convention is to test the oldest and newest + supported versions; intermediate versions do not need individual matrix entries. + +### Python+C repos (cibuildwheel) + +**`[tool.cibuildwheel]` in `pyproject.toml`**: + +- Update the `build` list, e.g. `["cp311-*", "cp312-*", "cp313-*"]`. This is what + controls which wheels are built — it is not inferred from the classifiers. +- When adding a new Python version, check that it is supported by the pinned + `cibuildwheel` release in the `wheels` dependency group. Pre-release Python versions + may require a newer cibuildwheel or explicit opt-in. +- After updating, push to the `test-publish` branch and verify that all new wheels + build and pass their smoke tests before making a release. Wheel builds are not + otherwise exercised in CI between releases, so this step is important. + +### Workflow files + +Check all `.github/workflows/` files in the repo for any explicit references to the +old minimum Python version and update them to the new minimum. In particular: + +- The `python-version` input passed to `python-c-tests.yml`, `python-packaging.yml`, + and `build-wheels.yml` should always match the current minimum supported Python. + (`lint.yml` does not need this — it never installs the package.) +- Any bespoke workflow steps that install or invoke a specific Python version directly. + ## Releases Python releases use the `wheels.yml` workflow via the Trusted Publisher PyPI mechanism. From e09f31576cb09426c47047141e7818474deb0f83 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 6 Mar 2026 14:35:21 +0000 Subject: [PATCH 098/106] Update with missing details --- repo_administration.md | 57 ++++++++++++++++++++++++++++++++---------- 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/repo_administration.md b/repo_administration.md index 8640c9c..3ba8093 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -24,9 +24,9 @@ core tskit-dev Python packages. easy to work across the ecosystem and reducing the maintenance burden of each individual repo. - **Centralised CI logic.** Shared workflows and composite actions in this repo - (`tskit-dev/.github`) avoid duplication. Individual repos call into them via - `uses: tskit-dev/.github/.github/workflows/.yml@main` or - `uses: tskit-dev/.github/.github/actions/@main`. + (`tskit-dev/.github`) avoid duplication. Individual repos call shared workflows via + a pinned version tag, e.g. `uses: tskit-dev/.github/.github/workflows/.yml@v14` + (see [CI workflows](#ci-workflows) for the versioning policy). - **Reproducibility.** All dependencies are managed exclusively with `uv` and locked via committed `uv.lock` files. Lint tools are pinned to exact versions so results are identical locally and in CI. @@ -53,18 +53,17 @@ The rules differ slightly between these categories, as described below. ### Python+C repos (external C library) -The Python package and its `pyproject.toml` live in a `python/` subdirectory. -The standalone C library lives in `c/`. Tests are in `python/tests/`. `prek.toml` -and `uv.lock` live at the root. +The Python package and its `pyproject.toml` and `uv.lock` live in a `python/` +subdirectory. The standalone C library lives in `c/`. +`prek.toml` lives at the root. Documentation (`docs/`) lives at the repository root and includes a `doxygen/` subdirectory for generating C API docs. ### Python+C repos (internal C library) -The Python package and its `pyproject.toml` live in a `python/` subdirectory. -The C extension code lives in `lib/` within the `python/` subdirectory. Tests -are in `python/tests/`. `prek.toml` and `uv.lock` live at the root. +The Python package and its `pyproject.toml` and `uv.lock` live in the +root, along with `prek.toml`. The C extension code lives in `lib/`. There is no separate C API documentation or independent C release workflow. @@ -73,7 +72,6 @@ There is no separate C API documentation or independent C release workflow. All shared workflows that accept a `pyproject-directory` input should be passed `python` for these repos. - ## Dependency management We use `uv` exclusively for Python package management. @@ -124,8 +122,23 @@ a controlled fashion. ## CI workflows -Each repo calls shared workflows defined in this repository. The available -workflows are: +Each repo calls shared workflows defined in this repository. Shared workflows are +versioned via git tags on this repository (`v5`, `v6`, … `v14`, etc.). Calling repos +pin to a specific tag, e.g.: + +```yaml +uses: tskit-dev/.github/.github/workflows/python-tests.yml@v14 +``` + +When a change is made to a shared workflow or composite action in this repo, create a +new tag (incrementing the version number) to publish the change. While developing +the changes you can use `@main` instead of the tagged version to pick up the +latest changes. Calling repos must +then update their pin to pick it up — this is a deliberate opt-in. To update a repo, +find all occurrences of the old tag in its `.github/workflows/` files and bump them to +the new version. + +The available workflows are: | Workflow | Purpose | |---|---| @@ -226,6 +239,23 @@ the `c-python` flag. **`c-tests.yml`** runs the C unit tests under gcc (with coverage), clang, and valgrind. Coverage is uploaded with the `C` flag. +### Required secrets and environments + +Each repo needs the following configured before releases, coverage uploads +and website rebuild triggers to work: + +- **`CODECOV_TOKEN`** GitHub Actions secret — for coverage uploads (see [Test coverage](#test-coverage)). +- **`ADMINBOT_TOKEN`** GitHub Actions secret — used by the shared `docs.yml` workflow to + fire a `repository_dispatch` to tskit-site after a successful docs build on `main`. + Without this, merging documentation changes will not trigger a site rebuild. +- **`release` GitHub Actions environment** — required by `wheels.yml` for the Trusted + Publisher OIDC token (`id-token: write`). Create this environment in the repo's + Settings → Environments. +- **Trusted Publisher on PyPI and TestPyPI** — configure a Trusted Publisher entry on + both [pypi.org](https://pypi.org) and [test.pypi.org](https://test.pypi.org) pointing + to the repo and the `release` environment. Without this, the `wheels.yml` upload steps + will fail even if the workflow runs successfully. + ## Test coverage @@ -292,6 +322,7 @@ Python releases use the `wheels.yml` workflow via the Trusted Publisher PyPI mec Because Trusted Publisher requires the upload to originate from the source repository, this workflow is not shared — each repo maintains its own copy. + Python+C repos additionally use the shared `build-wheels.yml` workflow, which uses `cibuildwheel` to build binary wheels across Linux, macOS, and Windows. All configuration lives in `pyproject.toml` under `[tool.cibuildwheel]`. @@ -360,7 +391,7 @@ Follow the standard Python release process above, with two differences: - In step 1, also set the version in `python//_version.py` (these repos do not use `setuptools_scm`). - In step 4, also bump the version in `python//_version.py` to - `MAJOR.MINOR.PATCH.dev1`. + `MAJOR.MINOR.PATCH.dev0`. ## The tskit.dev website From f5aa0cddad7d40e6b4d82b5f02ed1ccd9bc3a05f Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 6 Mar 2026 16:21:02 +0000 Subject: [PATCH 099/106] Remove Python version from lint workflow --- .github/workflows/lint.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b014d21..a4c9874 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -11,7 +11,7 @@ on: required: false type: string default: '.' - + jobs: prek: name: Lint @@ -23,20 +23,19 @@ jobs: access_token: ${{ github.token }} - uses: actions/checkout@v4.2.2 - with: + with: submodules: true - name: Install uv and set the python version uses: astral-sh/setup-uv@v6 with: - python-version: 3.11 version: "0.10.2" - - name: Install lint group + - name: Install lint group # Nore there is some subtlety here in using the --only-group approach. # We **do not** install the local package, which means we don't need # any system dependencies, but it can lead to oddities. - run: uv sync --locked --project=${{ inputs.pyproject-directory }} --only-group lint + run: uv sync --locked --project=${{ inputs.pyproject-directory }} --only-group lint - name: Run prek run: | From 5466d6dcd8bd63f1de94a7e95174e95fb34e0043 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 6 Mar 2026 16:29:59 +0000 Subject: [PATCH 100/106] Remove unused Python version and update uv --- .github/actions/build-docs/action.yml | 2 +- .github/workflows/build-wheels.yml | 10 ++++------ .github/workflows/c-tests.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/python-c-tests.yml | 5 ++--- .github/workflows/python-packaging.yml | 5 ++--- .github/workflows/python-tests.yml | 2 +- 7 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.github/actions/build-docs/action.yml b/.github/actions/build-docs/action.yml index b0788db..6f1fb36 100644 --- a/.github/actions/build-docs/action.yml +++ b/.github/actions/build-docs/action.yml @@ -60,7 +60,7 @@ runs: - name: Install uv uses: astral-sh/setup-uv@v6 with: - version: "0.8.15" + version: "0.10.0" - name: Install doc deps shell: bash diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index bd58b74..42e973d 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -23,11 +23,10 @@ jobs: fetch-depth: 0 submodules: true - - name: Install uv and set the python version + - name: Install uv uses: astral-sh/setup-uv@v6 with: - python-version: "3.12" - version: "0.8.15" + version: "0.10.0" - name: Build sdist run: uv build --project=${{ inputs.pyproject-directory }} --sdist -o dist @@ -69,11 +68,10 @@ jobs: shell: bash run: cp *.tar.gz sdist.tar.gz - - name: Install uv and set the python version + - name: Install uv uses: astral-sh/setup-uv@v6 with: - python-version: "3.12" - version: "0.8.15" + version: "0.10.0" - name: Install cibuildwheel run: uv sync --project=${{ inputs.pyproject-directory }} --locked --only-group wheels diff --git a/.github/workflows/c-tests.yml b/.github/workflows/c-tests.yml index b706657..453962f 100644 --- a/.github/workflows/c-tests.yml +++ b/.github/workflows/c-tests.yml @@ -38,7 +38,7 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v6 with: - version: "0.10.2" + version: "0.10.0" - name: Install uv deps run: | diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a4c9874..073b69d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -29,7 +29,7 @@ jobs: - name: Install uv and set the python version uses: astral-sh/setup-uv@v6 with: - version: "0.10.2" + version: "0.10.0" - name: Install lint group # Nore there is some subtlety here in using the --only-group approach. diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index d0319a2..5a58c56 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -40,11 +40,10 @@ jobs: run: | sudo apt-get update && sudo apt-get install -y ${{ inputs.additional-apt-packages }} - - name: Install uv and set the python version + - name: Install uv uses: astral-sh/setup-uv@v6 with: - python-version: 3.11 - version: "0.8.15" + version: "0.10.0" - name: Install Python dependencies run: uv sync --project=${{ inputs.pyproject-directory }} --locked --group test --no-default-groups diff --git a/.github/workflows/python-packaging.yml b/.github/workflows/python-packaging.yml index b265bdb..8e92db4 100644 --- a/.github/workflows/python-packaging.yml +++ b/.github/workflows/python-packaging.yml @@ -37,11 +37,10 @@ jobs: if: ${{ inputs.additional-apt-packages != '' }} run: sudo apt-get update && sudo apt-get install -y ${{ inputs.additional-apt-packages }} - - name: Install uv and set the python version + - name: Install uv uses: astral-sh/setup-uv@v6 with: - python-version: "3.12" - version: "0.8.15" + version: "0.10.0" - name: Install packaging deps working-directory: ${{ inputs.pyproject-directory }} diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 5d0c830..9a20cae 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -47,7 +47,7 @@ jobs: uses: astral-sh/setup-uv@v6 with: python-version: ${{ inputs.python-version }} - version: "0.8.15" + version: "0.10.0" - name: Install test deps run: uv sync --project=${{ inputs.pyproject-directory }} --locked --group test --no-default-groups From 4562169a6534229e58d920cced2afae63432a84b Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 6 Mar 2026 16:37:11 +0000 Subject: [PATCH 101/106] Minor tweaks to lint file --- .github/workflows/lint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 073b69d..e8dede7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,13 +26,13 @@ jobs: with: submodules: true - - name: Install uv and set the python version + - name: Install uv uses: astral-sh/setup-uv@v6 with: version: "0.10.0" - name: Install lint group - # Nore there is some subtlety here in using the --only-group approach. + # Note there is some subtlety here in using the --only-group approach. # We **do not** install the local package, which means we don't need # any system dependencies, but it can lead to oddities. run: uv sync --locked --project=${{ inputs.pyproject-directory }} --only-group lint From fe72d4dcb66cab926ed8876b30b6b139b0fdcabc Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 6 Mar 2026 16:58:21 +0000 Subject: [PATCH 102/106] Update docs for v15 tag and dropped python-version inputs --- repo_administration.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/repo_administration.md b/repo_administration.md index 3ba8093..9045e32 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -25,7 +25,7 @@ core tskit-dev Python packages. individual repo. - **Centralised CI logic.** Shared workflows and composite actions in this repo (`tskit-dev/.github`) avoid duplication. Individual repos call shared workflows via - a pinned version tag, e.g. `uses: tskit-dev/.github/.github/workflows/.yml@v14` + a pinned version tag, e.g. `uses: tskit-dev/.github/.github/workflows/.yml@v15` (see [CI workflows](#ci-workflows) for the versioning policy). - **Reproducibility.** All dependencies are managed exclusively with `uv` and locked via committed `uv.lock` files. Lint tools are pinned to exact versions @@ -127,7 +127,7 @@ versioned via git tags on this repository (`v5`, `v6`, … `v14`, etc.). Calling pin to a specific tag, e.g.: ```yaml -uses: tskit-dev/.github/.github/workflows/python-tests.yml@v14 +uses: tskit-dev/.github/.github/workflows/python-tests.yml@v15 ``` When a change is made to a shared workflow or composite action in this repo, create a @@ -311,9 +311,6 @@ affected repo: Check all `.github/workflows/` files in the repo for any explicit references to the old minimum Python version and update them to the new minimum. In particular: -- The `python-version` input passed to `python-c-tests.yml`, `python-packaging.yml`, - and `build-wheels.yml` should always match the current minimum supported Python. - (`lint.yml` does not need this — it never installs the package.) - Any bespoke workflow steps that install or invoke a specific Python version directly. ## Releases From 64423b91ec078ee615a3ed3a296cb3854dc18cfe Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Mon, 9 Mar 2026 09:39:36 +0000 Subject: [PATCH 103/106] Add conda-forge packages section to repo administration docs --- repo_administration.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/repo_administration.md b/repo_administration.md index 9045e32..0ce0bda 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -11,6 +11,7 @@ - [Releases](#releases) - [Standard Python release process](#standard-python-release-process) - [tskit and kastore releases](#tskit-and-kastore-releases) +- [conda-forge packages](#conda-forge-packages) - [The tskit.dev website](#the-tskitdev-website) ## Overview @@ -391,6 +392,29 @@ Follow the standard Python release process above, with two differences: `MAJOR.MINOR.PATCH.dev0`. +## conda-forge packages + +Several tskit-dev packages are distributed via conda-forge: +tskit, msprime, kastore, tstrait, tszip, and tsinfer. +Each has a feedstock repository at `https://github.com/conda-forge/-feedstock`. + +### How conda-forge updates work + +conda-forge bots monitor PyPI and automatically open a PR on the feedstock repository +when a new release is published. This PR updates the version number and source hash in +`recipe/meta.yaml`. In most cases the PR can simply be merged with no manual +intervention — the bot handles the version bump and rebuilds. + +### What requires manual attention + +Dependency changes are **not** picked up automatically. Whenever a package's +dependencies change (additions, removals, or version constraint changes), the +corresponding feedstock's `recipe/meta.yaml` must be updated manually: + +- `run:` requirements must reflect the current `[project.dependencies]` in + `pyproject.toml`. + + ## The tskit.dev website The [tskit-site](https://github.com/tskit-dev/tskit-site) repository is the source for From 5c1af62e14604528efac476e7f9b245d29f87d84 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 26 Mar 2026 21:28:38 +0000 Subject: [PATCH 104/106] Add fix for windows symlinks --- .github/workflows/python-tests.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 9a20cae..27c3d53 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -20,6 +20,11 @@ on: description: 'Directory to calculate coverage for' required: true type: string + fix-lwt-symlink: + description: 'Fix the lwt symlinks on windows' + required: false + type: string + default: "false" jobs: test: @@ -39,6 +44,12 @@ jobs: with: submodules: true + - name: Fix windows symlinks + if: ${{ inputs.fix-lwt-symlink == 'true' && runner.os == 'Windows' }} + run: | + rm lwt_interface + cp -r --dereference git-submodules/tskit/python/lwt_interface ./lwt_interface + - name: Install additional APT packages if: ${{ inputs.additional-apt-packages != '' }} run: sudo apt-get update && sudo apt-get install -y ${{ inputs.additional-apt-packages }} From 48464f9a373c136be4fd4095d8de476a3734dced Mon Sep 17 00:00:00 2001 From: peter Date: Thu, 14 May 2026 11:23:51 -0700 Subject: [PATCH 105/106] doc clarifications --- repo_administration.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/repo_administration.md b/repo_administration.md index 0ce0bda..c9a9bc2 100644 --- a/repo_administration.md +++ b/repo_administration.md @@ -345,12 +345,17 @@ maintain version numbers manually. release artifacts and publishes them to TestPyPI. Check the "Publish Python release" action succeeds. This step is especially important for repos with binary wheels (msprime, tsinfer, tskit, kastore) as the wheel-building step is **not tested** between releases. + To find the results of the build action, go to the "Actions" tab in github, + and look for the action associated with this branch; next go to + `https://test.pypi.org/project//#history` to see if the upload worked. 3. Once the TestPyPI upload succeeds, delete the `test-publish` branch. Go to the - "Releases" section on GitHub and click "Draft new release". Enter the version number + "Releases" section on GitHub and click "Draft new release" (see the names of + previous releases for naming convention). Enter the version number in the tag box and click "create new tag" (the tag is created when the release is - published). Fill in the release body with the CHANGELOG contents (the "latest" docs - on tskit.dev have most of the formatting done already). Click "Publish release" and - confirm the "Publish Python release" action succeeds on PyPI. + published). Fill in the release body with the CHANGELOG contents (you can copy-paste + from the CHANGELOG in the "latest" docs on tskit.dev to get the formatting right). + Click "Publish release" and confirm the "Publish Python release" action succeeds + (under the Actions tab in github) and that it is successfully available on PyPI. 4. Open a post-release PR that opens a new section in the CHANGELOG. ### tskit and kastore releases From 268974db6e0a77bdfa966bcd60da8ffa57738543 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 16 Jun 2026 22:08:52 -0700 Subject: [PATCH 106/106] update codecov action --- .github/workflows/c-tests.yml | 2 +- .github/workflows/python-c-tests.yml | 2 +- .github/workflows/python-tests.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/c-tests.yml b/.github/workflows/c-tests.yml index 453962f..2a386de 100644 --- a/.github/workflows/c-tests.yml +++ b/.github/workflows/c-tests.yml @@ -58,7 +58,7 @@ jobs: run: ninja -C build-cov coverage-xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5.4.0 + uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} flags: C diff --git a/.github/workflows/python-c-tests.yml b/.github/workflows/python-c-tests.yml index 5a58c56..03c1ba7 100644 --- a/.github/workflows/python-c-tests.yml +++ b/.github/workflows/python-c-tests.yml @@ -64,7 +64,7 @@ jobs: run: uv tool run gcovr --clover coverage.xml --txt - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5.4.0 + uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} files: coverage.xml diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 27c3d53..212aaa2 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -70,7 +70,7 @@ jobs: ${{ inputs.pyproject-directory }}/tests - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5.5.2 + uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true