From a67273b9fb80157109304f5c5ef6222e9a4fce8b Mon Sep 17 00:00:00 2001 From: BlueGlassBlock Date: Mon, 27 Feb 2023 16:01:18 +0800 Subject: [PATCH 1/7] Update to Python 3.11 --- .github/scripts/generate_tx_config.py | 119 ++++++++++++-------------- .github/scripts/prepare.sh | 2 +- .github/scripts/transifex_pull.py | 38 -------- .github/scripts/update.sh | 15 +--- .github/workflows/python-310.yml | 2 +- .github/workflows/python-311.yml | 29 +++++++ .github/workflows/python-37.yml | 2 +- .github/workflows/python-38.yml | 2 +- .github/workflows/python-39.yml | 2 +- README.rst | 3 + 10 files changed, 93 insertions(+), 121 deletions(-) delete mode 100755 .github/scripts/transifex_pull.py create mode 100644 .github/workflows/python-311.yml diff --git a/.github/scripts/generate_tx_config.py b/.github/scripts/generate_tx_config.py index 0e72833f1..b21e099ee 100755 --- a/.github/scripts/generate_tx_config.py +++ b/.github/scripts/generate_tx_config.py @@ -1,76 +1,67 @@ -#!/usr/bin/env python3 -import glob -import json -import os +"""Please note that this script requires a Transifex API token to run.""" +import subprocess +from functools import partial +from pathlib import Path import re -import sys -import urllib.request +run = partial(subprocess.run, check=True) -def list_resources(token, project): - auth_handler = urllib.request.HTTPBasicAuthHandler() - auth_handler.add_password( - realm="api", uri="https://api.transifex.com/", user="api", passwd=token + +def init_project(): + run(["tx", "init"]) + + +def add_files(version: str): + run( + [ + "tx", + "add", + "remote", + "--file-filter", + "trans//.", + f"https://www.transifex.com/python-doc/{version}/dashboard/", + ] ) - opener = urllib.request.build_opener(auth_handler) - urllib.request.install_opener(opener) - next_ = ( - "https://api.transifex.com/organizations/python-doc/projects/" - + project - + "/resources/" + + +FILTER_PATTERN = re.compile( + r"^(?Pfile_filter( *)=( *))(?P.+)$", re.MULTILINE +) + + +def name_replacer(match: re.Match[str]): + prefix, resource = match.group("prefix", "resource") + override_prefix = prefix.replace("file_filter", "trans.zh_CN") + override_resource = ( + resource.replace("trans//", "") + .replace("--", "/") + .replace("glossary_", "glossary") + .replace("_", ".") ) - resources = [] - while True: - resp = urllib.request.urlopen(next_) - result = json.loads(resp.read().decode("utf-8")) - resources.extend([i["slug"] for i in result]) - link = re.findall('<([^<]*)>; rel="next"', resp.getheader("Link") or "") - if not link: - break - next_ = link[0] - return resources - - -def render_config(doc_dir, project, resources): - os.chdir(doc_dir) - tpl = """ - -[{project}.{resource}] -trans.zh_CN = {filename} -source_lang = en -type = PO""" - conf = """[main] -host = https://www.transifex.com""" - for resource in sorted(resources): - if resource == "glossary_": - filename = "glossary.po" - elif resource == "sphinx": - filename = "sphinx.po" - else: - pattern = resource.replace("--", "/").replace("_", "?") - matches = glob.glob(pattern + ".rst") - if len(matches) == 0: - print("missing", resource, file=sys.stderr) - continue - elif len(matches) == 1: - filename = matches[0].replace(".rst", ".po") - else: - print("multi match", resource, pattern, matches, file=sys.stderr) - conf += tpl.format(project=project, resource=resource, filename=filename) - return conf + return f"{prefix}{resource}\n{override_prefix}{override_resource}" + + +def patch_config(): + tx_config_path = Path(".tx", "config") + + config_content = tx_config_path.read_text("utf-8") + config_content = FILTER_PATTERN.sub(name_replacer, config_content) + tx_config_path.write_text(config_content, "utf-8") if __name__ == "__main__": - import argparse + from argparse import ArgumentParser + import os + + parser = ArgumentParser() - parser = argparse.ArgumentParser() parser.add_argument("--token") - parser.add_argument("--project") - parser.add_argument("--doc-dir") - args = parser.parse_args() + parser.add_argument("--version") + + params = parser.parse_args() - resources = list_resources(args.token, args.project) - conf = render_config(args.doc_dir, args.project, resources) - print(conf) + os.environ["TX_TOKEN"] = params.token -# vim: set et ts=4 sw=4 sts=4: + init_project() + add_files(params.version) + patch_config() diff --git a/.github/scripts/prepare.sh b/.github/scripts/prepare.sh index d16dbc056..cf1326646 100755 --- a/.github/scripts/prepare.sh +++ b/.github/scripts/prepare.sh @@ -6,6 +6,6 @@ git clone --depth=1 --branch="$VERSION" https://github.com/python/cpython cpytho git clone --branch="$VERSION" https://github.com/"$GITHUB_REPOSITORY" docs pip3 install --user setuptools -pip3 install --user transifex-client +curl -o- https://raw.githubusercontent.com/transifex/cli/master/install.sh | bash sudo apt-get update sudo apt-get install -y python3-venv diff --git a/.github/scripts/transifex_pull.py b/.github/scripts/transifex_pull.py deleted file mode 100755 index 66c4046d9..000000000 --- a/.github/scripts/transifex_pull.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python3 - -from txclib import project -from txclib.utils import perform_parallel_requests - - -def pull(path, lang): - skip_decode = False - params = {} - parallel = True - - prj = project.Project(path) - resource_list = prj.get_chosen_resources([]) - for resource in resource_list: - project_slug, resource_slug = resource.split(".", 1) - host = prj.get_resource_host(resource) - prj._set_url_info(host=host, project=project_slug, resource=resource_slug) - - files = prj.get_resource_files(resource) - url = prj._get_url_by_pull_mode(None) - local_file = files.get(lang) - prj.do_url_request( - url, - language=lang, - skip_decode=skip_decode, - params=params, - parallel=parallel, - callback=prj._save_file, - callback_args={"local_file": local_file}, - ) - - perform_parallel_requests() - - -if __name__ == "__main__": - import os - - pull(os.getcwd(), os.getenv("LOCALE", "zh_CN")) diff --git a/.github/scripts/update.sh b/.github/scripts/update.sh index 544a31ae5..8424d2901 100755 --- a/.github/scripts/update.sh +++ b/.github/scripts/update.sh @@ -1,18 +1,5 @@ #!/bin/bash -set -ex - -script_dir="$(dirname "$(realpath "$0")")" - -if [[ -n "$TRANSIFEX_APIKEY" ]]; then - cat > ~/.transifexrc << EOF -[https://www.transifex.com] -api_hostname = https://api.transifex.com -hostname = https://www.transifex.com -password = $TRANSIFEX_APIKEY -username = api -EOF -fi cd docs || exit 1 -"$script_dir"/transifex_pull.py +tx pull --languages "$LOCALE" diff --git a/.github/workflows/python-310.yml b/.github/workflows/python-310.yml index 54037983e..5524337ef 100644 --- a/.github/workflows/python-310.yml +++ b/.github/workflows/python-310.yml @@ -20,7 +20,7 @@ jobs: - name: update run: .github/scripts/update.sh env: - TRANSIFEX_APIKEY: ${{ secrets.TRANSIFEX_APIKEY }} + TX_TOKEN: ${{ secrets.TRANSIFEX_APIKEY }} - name: build run: .github/scripts/build.sh - name: commit diff --git a/.github/workflows/python-311.yml b/.github/workflows/python-311.yml new file mode 100644 index 000000000..e37354ae2 --- /dev/null +++ b/.github/workflows/python-311.yml @@ -0,0 +1,29 @@ +name: python-310 + +on: + push: + branches: + - master + schedule: + - cron: "8 * * * *" + +jobs: + sync: + runs-on: ubuntu-latest + env: + LOCALE: zh_CN + VERSION: "3.11" + steps: + - uses: actions/checkout@v2 + - name: prepare + run: .github/scripts/prepare.sh + - name: update + run: .github/scripts/update.sh + env: + TX_TOKEN: ${{ secrets.TRANSIFEX_APIKEY }} + - name: build + run: .github/scripts/build.sh + - name: commit + run: .github/scripts/commit.sh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/python-37.yml b/.github/workflows/python-37.yml index c35d61d2b..6864df933 100644 --- a/.github/workflows/python-37.yml +++ b/.github/workflows/python-37.yml @@ -20,7 +20,7 @@ jobs: - name: update run: .github/scripts/update.sh env: - TRANSIFEX_APIKEY: ${{ secrets.TRANSIFEX_APIKEY }} + TX_TOKEN: ${{ secrets.TRANSIFEX_APIKEY }} - name: build run: .github/scripts/build.sh - name: commit diff --git a/.github/workflows/python-38.yml b/.github/workflows/python-38.yml index a99f0e20e..35948544c 100644 --- a/.github/workflows/python-38.yml +++ b/.github/workflows/python-38.yml @@ -20,7 +20,7 @@ jobs: - name: update run: .github/scripts/update.sh env: - TRANSIFEX_APIKEY: ${{ secrets.TRANSIFEX_APIKEY }} + TX_TOKEN: ${{ secrets.TRANSIFEX_APIKEY }} - name: build run: .github/scripts/build.sh - name: commit diff --git a/.github/workflows/python-39.yml b/.github/workflows/python-39.yml index 76a6793bb..4b16958a5 100644 --- a/.github/workflows/python-39.yml +++ b/.github/workflows/python-39.yml @@ -20,7 +20,7 @@ jobs: - name: update run: .github/scripts/update.sh env: - TRANSIFEX_APIKEY: ${{ secrets.TRANSIFEX_APIKEY }} + TX_TOKEN: ${{ secrets.TRANSIFEX_APIKEY }} - name: build run: .github/scripts/build.sh - name: commit diff --git a/README.rst b/README.rst index 4ff062bc4..cf08f1294 100644 --- a/README.rst +++ b/README.rst @@ -12,6 +12,9 @@ Maintained versions: * - Version - Sync status - Translation progress + * - `3.11 `_ + - .. image:: https://github.com/python/python-docs-zh-cn/workflows/python-311/badge.svg + :target: https://github.com/python/python-docs-zh-cn/actions?workflow=python-311 * - `3.10 `_ - .. image:: https://github.com/python/python-docs-zh-cn/workflows/python-310/badge.svg :target: https://github.com/python/python-docs-zh-cn/actions?workflow=python-310 From 8602d5e6b7413731814e85181ab97032d384ccb7 Mon Sep 17 00:00:00 2001 From: Nyuan Zhang Date: Mon, 27 Feb 2023 16:20:58 +0800 Subject: [PATCH 2/7] Update python-311.yml --- .github/workflows/python-311.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-311.yml b/.github/workflows/python-311.yml index e37354ae2..d2f7aee35 100644 --- a/.github/workflows/python-311.yml +++ b/.github/workflows/python-311.yml @@ -1,11 +1,11 @@ -name: python-310 +name: python-311 on: push: branches: - master schedule: - - cron: "8 * * * *" + - cron: "11 * * * *" jobs: sync: From 3554fc2397bf0d7975b5368b569ce36a8e81dc71 Mon Sep 17 00:00:00 2001 From: BlueGlassBlock Date: Mon, 27 Feb 2023 17:59:07 +0800 Subject: [PATCH 3/7] Fix transifex cli position --- .github/scripts/update.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/update.sh b/.github/scripts/update.sh index 8424d2901..41279d019 100755 --- a/.github/scripts/update.sh +++ b/.github/scripts/update.sh @@ -1,5 +1,5 @@ #!/bin/bash - +tx=$(realpath ./tx) cd docs || exit 1 -tx pull --languages "$LOCALE" +$tx pull --languages "$LOCALE" From c259ee1c2ba0576ceedbb67aeac979c85c9ebbd2 Mon Sep 17 00:00:00 2001 From: BlueGlassBlock Date: Tue, 28 Feb 2023 14:11:44 +0800 Subject: [PATCH 4/7] Fix config generation script --- .github/scripts/generate_tx_config.py | 35 +++++++++++++++++++-------- .github/scripts/update.sh | 2 +- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/.github/scripts/generate_tx_config.py b/.github/scripts/generate_tx_config.py index b21e099ee..f021d4ac9 100755 --- a/.github/scripts/generate_tx_config.py +++ b/.github/scripts/generate_tx_config.py @@ -1,4 +1,5 @@ """Please note that this script requires a Transifex API token to run.""" +import glob import subprocess from functools import partial from pathlib import Path @@ -32,20 +33,33 @@ def add_files(version: str): def name_replacer(match: re.Match[str]): prefix, resource = match.group("prefix", "resource") override_prefix = prefix.replace("file_filter", "trans.zh_CN") - override_resource = ( + pattern = ( resource.replace("trans//", "") - .replace("--", "/") .replace("glossary_", "glossary") - .replace("_", ".") + .replace("--", "/") + .replace("_", "?") ) - return f"{prefix}{resource}\n{override_prefix}{override_resource}" - - -def patch_config(): + matches = list(glob.glob(pattern.replace(".po", ".rst"))) + if not matches: + print("missing", pattern) + return f"{prefix}{resource}\n{override_prefix}{pattern}" + elif len(matches) == 1: + filename = matches[0].replace(".rst", ".po").replace("\\", "/") + else: + raise ValueError("multi match", resource, pattern, matches) + return f"{prefix}{resource}\n{override_prefix}{filename}" + + +def patch_config(path: str): tx_config_path = Path(".tx", "config") config_content = tx_config_path.read_text("utf-8") + + cwd = os.getcwd() + os.chdir(path) config_content = FILTER_PATTERN.sub(name_replacer, config_content) + os.chdir(cwd) + tx_config_path.write_text(config_content, "utf-8") @@ -55,8 +69,9 @@ def patch_config(): parser = ArgumentParser() - parser.add_argument("--token") - parser.add_argument("--version") + parser.add_argument("--token", required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--doc-path", required=True) params = parser.parse_args() @@ -64,4 +79,4 @@ def patch_config(): init_project() add_files(params.version) - patch_config() + patch_config(params.doc_path) diff --git a/.github/scripts/update.sh b/.github/scripts/update.sh index 41279d019..bcdb2d7d9 100755 --- a/.github/scripts/update.sh +++ b/.github/scripts/update.sh @@ -2,4 +2,4 @@ tx=$(realpath ./tx) cd docs || exit 1 -$tx pull --languages "$LOCALE" +$tx pull --languages "$LOCALE" -t --use-git-timestamps From 6d039834cda6b52dd18cefea6cde0ca8a2134fed Mon Sep 17 00:00:00 2001 From: BlueGlassBlock Date: Tue, 28 Feb 2023 14:14:22 +0800 Subject: [PATCH 5/7] Add 3.11 to workflows --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index cf08f1294..d6847fc9c 100644 --- a/README.rst +++ b/README.rst @@ -15,11 +15,13 @@ Maintained versions: * - `3.11 `_ - .. image:: https://github.com/python/python-docs-zh-cn/workflows/python-311/badge.svg :target: https://github.com/python/python-docs-zh-cn/actions?workflow=python-311 + - .. image:: https://img.shields.io/badge/dynamic/json.svg?label=zh_CN&query=%24.zh_CN&url=http://gce.zhsj.me/python/310 + :target: https://www.transifex.com/python-doc/python-311/ * - `3.10 `_ - .. image:: https://github.com/python/python-docs-zh-cn/workflows/python-310/badge.svg :target: https://github.com/python/python-docs-zh-cn/actions?workflow=python-310 - .. image:: https://img.shields.io/badge/dynamic/json.svg?label=zh_CN&query=%24.zh_CN&url=http://gce.zhsj.me/python/310 - :target: https://www.transifex.com/python-doc/python-39/ + :target: https://www.transifex.com/python-doc/python-310/ * - `3.9 `_ - .. image:: https://github.com/python/python-docs-zh-cn/workflows/python-39/badge.svg :target: https://github.com/python/python-docs-zh-cn/actions?workflow=python-39 From c3d4b1a1b8502a649bfd502eee732cf25d723b0c Mon Sep 17 00:00:00 2001 From: BlueGlassBlock Date: Sat, 13 May 2023 21:36:05 +0800 Subject: [PATCH 6/7] chore: update 3.11 links and badges --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index d6847fc9c..53e6a6bed 100644 --- a/README.rst +++ b/README.rst @@ -12,10 +12,10 @@ Maintained versions: * - Version - Sync status - Translation progress - * - `3.11 `_ + * - `3.11 `_ - .. image:: https://github.com/python/python-docs-zh-cn/workflows/python-311/badge.svg :target: https://github.com/python/python-docs-zh-cn/actions?workflow=python-311 - - .. image:: https://img.shields.io/badge/dynamic/json.svg?label=zh_CN&query=%24.zh_CN&url=http://gce.zhsj.me/python/310 + - .. image:: https://img.shields.io/badge/dynamic/json.svg?label=zh_CN&query=%24.zh_CN&url=http://gce.zhsj.me/python/311 :target: https://www.transifex.com/python-doc/python-311/ * - `3.10 `_ - .. image:: https://github.com/python/python-docs-zh-cn/workflows/python-310/badge.svg From e01effbf98990f1f451c89ee77aa3af4089a5e33 Mon Sep 17 00:00:00 2001 From: BlueGlassBlock Date: Mon, 29 May 2023 13:14:03 +0800 Subject: [PATCH 7/7] chore: Add 3.12 to workflow --- .github/scripts/generate_tx_config.py | 17 ++++++++-------- .github/workflows/python-312.yml | 29 +++++++++++++++++++++++++++ README.rst | 15 +++++++++----- 3 files changed, 48 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/python-312.yml diff --git a/.github/scripts/generate_tx_config.py b/.github/scripts/generate_tx_config.py index f021d4ac9..ca3bb39e4 100755 --- a/.github/scripts/generate_tx_config.py +++ b/.github/scripts/generate_tx_config.py @@ -4,6 +4,7 @@ from functools import partial from pathlib import Path import re +import os run = partial(subprocess.run, check=True) @@ -12,7 +13,7 @@ def init_project(): run(["tx", "init"]) -def add_files(version: str): +def add_files(project_name: str): run( [ "tx", @@ -20,7 +21,7 @@ def add_files(version: str): "remote", "--file-filter", "trans//.", - f"https://www.transifex.com/python-doc/{version}/dashboard/", + f"https://www.transifex.com/python-doc/{project_name}/dashboard/", ] ) @@ -42,7 +43,7 @@ def name_replacer(match: re.Match[str]): matches = list(glob.glob(pattern.replace(".po", ".rst"))) if not matches: print("missing", pattern) - return f"{prefix}{resource}\n{override_prefix}{pattern}" + return f"{prefix}{resource}\n{override_prefix}{pattern.replace('?', '_')}" elif len(matches) == 1: filename = matches[0].replace(".rst", ".po").replace("\\", "/") else: @@ -65,18 +66,18 @@ def patch_config(path: str): if __name__ == "__main__": from argparse import ArgumentParser - import os parser = ArgumentParser() - parser.add_argument("--token", required=True) - parser.add_argument("--version", required=True) + parser.add_argument("--token", default="") + parser.add_argument("--project-name", required=True) parser.add_argument("--doc-path", required=True) params = parser.parse_args() - os.environ["TX_TOKEN"] = params.token + if params.token: + os.environ["TX_TOKEN"] = params.token init_project() - add_files(params.version) + add_files(params.project_name) patch_config(params.doc_path) diff --git a/.github/workflows/python-312.yml b/.github/workflows/python-312.yml new file mode 100644 index 000000000..51c5b3f6a --- /dev/null +++ b/.github/workflows/python-312.yml @@ -0,0 +1,29 @@ +name: python-312 + +on: + push: + branches: + - master + schedule: + - cron: "42 * * * *" + +jobs: + sync: + runs-on: ubuntu-latest + env: + LOCALE: zh_CN + VERSION: "3.12" + steps: + - uses: actions/checkout@v2 + - name: prepare + run: .github/scripts/prepare.sh + - name: update + run: .github/scripts/update.sh + env: + TX_TOKEN: ${{ secrets.TRANSIFEX_APIKEY }} + - name: build + run: .github/scripts/build.sh + - name: commit + run: .github/scripts/commit.sh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.rst b/README.rst index 53e6a6bed..604acc8ba 100644 --- a/README.rst +++ b/README.rst @@ -12,31 +12,36 @@ Maintained versions: * - Version - Sync status - Translation progress + * - `3.12 `_ + - .. image:: https://github.com/python/python-docs-zh-cn/workflows/python-312/badge.svg + :target: https://github.com/python/python-docs-zh-cn/actions?workflow=python-312 + - .. image:: https://img.shields.io/badge/dynamic/json.svg?label=zh_CN&query=%24.zh_CN&url=http://gce.zhsj.me/python/312 + :target: https://app.transifex.com/python-doc/python-newest/ * - `3.11 `_ - .. image:: https://github.com/python/python-docs-zh-cn/workflows/python-311/badge.svg :target: https://github.com/python/python-docs-zh-cn/actions?workflow=python-311 - .. image:: https://img.shields.io/badge/dynamic/json.svg?label=zh_CN&query=%24.zh_CN&url=http://gce.zhsj.me/python/311 - :target: https://www.transifex.com/python-doc/python-311/ + :target: https://app.transifex.com/python-doc/python-311/ * - `3.10 `_ - .. image:: https://github.com/python/python-docs-zh-cn/workflows/python-310/badge.svg :target: https://github.com/python/python-docs-zh-cn/actions?workflow=python-310 - .. image:: https://img.shields.io/badge/dynamic/json.svg?label=zh_CN&query=%24.zh_CN&url=http://gce.zhsj.me/python/310 - :target: https://www.transifex.com/python-doc/python-310/ + :target: https://app.transifex.com/python-doc/python-310/ * - `3.9 `_ - .. image:: https://github.com/python/python-docs-zh-cn/workflows/python-39/badge.svg :target: https://github.com/python/python-docs-zh-cn/actions?workflow=python-39 - .. image:: https://img.shields.io/badge/dynamic/json.svg?label=zh_CN&query=%24.zh_CN&url=http://gce.zhsj.me/python/39 - :target: https://www.transifex.com/python-doc/python-39/ + :target: https://app.transifex.com/python-doc/python-39/ * - `3.8 `_ - .. image:: https://github.com/python/python-docs-zh-cn/workflows/python-38/badge.svg :target: https://github.com/python/python-docs-zh-cn/actions?workflow=python-38 - .. image:: https://img.shields.io/badge/dynamic/json.svg?label=zh_CN&query=%24.zh_CN&url=http://gce.zhsj.me/python/38 - :target: https://www.transifex.com/python-doc/python-38/ + :target: https://app.transifex.com/python-doc/python-38/ * - `3.7 `_ - .. image:: https://github.com/python/python-docs-zh-cn/workflows/python-37/badge.svg :target: https://github.com/python/python-docs-zh-cn/actions?workflow=python-37 - .. image:: https://img.shields.io/badge/dynamic/json.svg?label=zh_CN&query=%24.zh_CN&url=http://gce.zhsj.me/python/37 - :target: https://www.transifex.com/python-doc/python-37/ + :target: https://app.transifex.com/python-doc/python-37/ Documentation Contribution Agreement ------------------------------------