diff --git a/.appveyor.yml b/.appveyor.yml index de17f8ba..ce025819 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -16,6 +16,10 @@ environment: python_version: 3.10.6 - python: 310-x64 python_version: 3.10.6 + - python: 311 + python_version: 3.11.2 + - python: 311-x64 + python_version: 3.11.2 install: - ps: | diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml index 191e2001..886bd8c9 100644 --- a/.github/workflows/linuxbrew.yml +++ b/.github/workflows/linuxbrew.yml @@ -3,26 +3,28 @@ on: [push, pull_request] jobs: linuxbrew: runs-on: ubuntu-latest + strategy: + matrix: + python: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - name: Install build dependencies + - name: Install brew run: | sudo apt install -y build-essential procps curl file git /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - test -d ~/.linuxbrew && eval $(~/.linuxbrew/bin/brew shellenv) - test -r ~/.bash_profile && echo "eval \$($(brew --prefix)/bin/brew shellenv)" >>~/.bash_profile - echo "eval \$($(brew --prefix)/bin/brew shellenv)" >>~/.profile + echo "/home/linuxbrew/.linuxbrew/bin" >> $GITHUB_PATH + - name: Install build dependencies + run: | brew update - brew install python gcc libxml2 libxmlsec1 pkg-config + brew install python@${{ matrix.python }} gcc libxml2 libxmlsec1 pkg-config + echo "/home/linuxbrew/.linuxbrew/opt/python@${{ matrix.python }}/libexec/bin" >> $GITHUB_PATH + - name: Install python dependencies + run: | pip3 install --upgrade setuptools wheel build - ln -s $(brew --prefix)/bin/gcc-12 $(brew --prefix)/bin/gcc-5 - ls -l $(brew --prefix)/bin/gcc* - name: Build linux_x86_64 wheel run: | + export CFLAGS="-I$(brew --prefix)/include" + export LDFLAGS="-L$(brew --prefix)/lib" python3 -m build rm -rf build/ - name: Install test dependencies diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index 4db5e306..6d0548e8 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -5,7 +5,8 @@ jobs: runs-on: macos-latest strategy: matrix: - python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10"] + python: [3.5, 3.6, 3.7, 3.8, 3.9, "3.10", "3.11"] + static_deps: ["static", ""] steps: - uses: actions/checkout@v3 - name: Setup Python @@ -21,7 +22,10 @@ jobs: CC: clang CFLAGS: "-fprofile-instr-generate -fcoverage-mapping" LDFLAGS: "-fprofile-instr-generate -fcoverage-mapping" + PYXMLSEC_STATIC_DEPS: ${{ matrix.static_deps }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + export PKG_CONFIG_PATH="$(brew --prefix)/opt/libxml2/lib/pkgconfig" python -m build rm -rf build/ - name: Set environment variables @@ -40,5 +44,6 @@ jobs: - name: Report coverage to codecov run: | /Library/Developer/CommandLineTools/usr/bin/llvm-profdata merge -sparse ${{ env.LLVM_PROFILE_FILE }} -output pyxmlsec.profdata - /Library/Developer/CommandLineTools/usr/bin/llvm-cov show ${{ env.PYXMLSEC_LIBFILE }} -instr-profile=pyxmlsec.profdata src > coverage.txt + /Library/Developer/CommandLineTools/usr/bin/llvm-cov show ${{ env.PYXMLSEC_LIBFILE }} --arch=$(uname -m) --instr-profile=pyxmlsec.profdata src > coverage.txt bash <(curl -s https://codecov.io/bash) -f coverage.txt + if: matrix.static_deps != 'static' diff --git a/.github/workflows/manylinux.yml b/.github/workflows/manylinux.yml index 867c17ba..a44776b3 100644 --- a/.github/workflows/manylinux.yml +++ b/.github/workflows/manylinux.yml @@ -1,28 +1,27 @@ name: manylinux on: [push, pull_request] jobs: - pep513: + manylinux: runs-on: ubuntu-latest strategy: matrix: - python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310] + python-abi: [cp36-cp36m, cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310, cp311-cp311] image: - - manylinux2010_x86_64 - - manylinux_2_24_x86_64 + - manylinux2014_x86_64 + - manylinux_2_28_x86_64 - musllinux_1_1_x86_64 - exclude: - - image: manylinux2010_x86_64 - python-abi: cp310-cp310 - - image: manylinux2010_i686 - python-abi: cp310-cp310 container: quay.io/pypa/${{ matrix.image }} steps: - uses: actions/checkout@v1 - - name: Install build dependencies + - name: Install python build dependencies run: | # https://github.com/actions/runner/issues/2033 chown -R $(id -u):$(id -g) $PWD /opt/python/${{ matrix.python-abi }}/bin/pip install --upgrade pip setuptools wheel build + - name: Install system build dependencies (manylinux) + run: | + yum install -y perl-core + if: contains(matrix.image, 'manylinux') - name: Set environment variables shell: bash run: | @@ -30,6 +29,7 @@ jobs: - name: Build linux_x86_64 wheel env: PYXMLSEC_STATIC_DEPS: true + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | /opt/python/${{ matrix.python-abi }}/bin/python -m build - name: Label manylinux wheel diff --git a/.github/workflows/opensuse-tumbleweed.yml b/.github/workflows/opensuse-tumbleweed.yml index d8bb8113..2f4caf49 100644 --- a/.github/workflows/opensuse-tumbleweed.yml +++ b/.github/workflows/opensuse-tumbleweed.yml @@ -6,14 +6,18 @@ jobs: container: opensuse/tumbleweed strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v1 - name: Install build dependencies run: | - zypper -n install -t pattern devel_basis + zypper refresh + zypper update + # The follwoing installs "devel_basis" pattern since installing the pattern fails because of few + # incompatibilty issues among packages + zypper -n install autoconf automake binutils bison cpp cpp13 flex gawk gcc gcc13 gdbm-devel gettext-runtime gettext-tools glibc-devel info kbd kbd-legacy libapparmor1 libasan8 libatomic1 libctf-nobfd0 libctf0 libdb-4_8 libfl-devel libfl2 libgdbm6 libgdbm_compat4 libgomp1 libhwasan0 libisl23 libitm1 libkmod2 liblsan0 libltdl7 libmpc3 libmpfr6 libseccomp2 libtextstyle0 libtool libtsan2 libubsan1 libxcrypt-devel libzio1 linux-glibc-devel m4 make makeinfo ncurses-devel pam-config patch perl perl-Text-Unidecode perl-base purge-kernels-service system-user-nobody systemd systemd-default-settings systemd-default-settings-branding-openSUSE systemd-presets-branding-openSUSE systemd-presets-common-SUSE tack update-alternatives zlib-devel PKGVER_NO_DOT=$(tr -d '.' <<< ${{ matrix.python-version }}) - zypper -n install git libxmlsec1-openssl1 xmlsec1-openssl-devel python${PKGVER_NO_DOT}-devel python${PKGVER_NO_DOT}-pip + zypper -n install git libxmlsec1-openssl1 xmlsec1-openssl-devel python${PKGVER_NO_DOT}-devel python${{ matrix.python-version }} -m venv .venv .venv/bin/python -m pip install --upgrade pip setuptools wheel - name: Build linux_x86_64 wheel @@ -22,7 +26,7 @@ jobs: rm -rf build/ - name: Install test dependencies run: | - .venv/bin/python -m pip install --upgrade -r requirements-test.txt + .venv/bin/python -m pip install --upgrade --no-binary=lxml -r requirements-test.txt .venv/bin/python -m pip install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ - name: Run tests run: | diff --git a/.github/workflows/sdist.yml b/.github/workflows/sdist.yml index fb13377a..e7c0f39d 100644 --- a/.github/workflows/sdist.yml +++ b/.github/workflows/sdist.yml @@ -5,10 +5,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Set up Python 3.10 + - name: Set up Python 3.11 uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Install build dependencies run: | pip install --upgrade pip setuptools wheel @@ -16,11 +16,9 @@ jobs: run: | python setup.py sdist - name: Install test dependencies - env: - PYXMLSEC_STATIC_DEPS: true run: | - pip install --upgrade -r requirements-test.txt - pip install black # for stub generation tests + sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl opensc softhsm2 libengine-pkcs11-openssl + pip install --upgrade -r requirements-test.txt --no-binary lxml pip install dist/xmlsec-$(python setup.py --version).tar.gz - name: Run tests run: | diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 00000000..ccd62cf8 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,124 @@ +name: Wheel build + +on: + release: + types: [created] + schedule: + # ┌───────────── minute (0 - 59) + # │ ┌───────────── hour (0 - 23) + # │ │ ┌───────────── day of the month (1 - 31) + # │ │ │ ┌───────────── month (1 - 12 or JAN-DEC) + # │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT) + # │ │ │ │ │ + - cron: "42 3 * * 4" + push: + pull_request: + workflow_dispatch: + +permissions: {} + +jobs: + sdist: + runs-on: ubuntu-latest + + permissions: + contents: write + + steps: + - uses: actions/checkout@v4.1.1 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5.0.0 + with: + python-version: "3.x" + + - name: Install build dependencies + run: | + pip install --upgrade pip setuptools wheel + + - name: Package source dist + run: python setup.py sdist + + - name: Install test dependencies + run: | + sudo apt-get update -y -q + sudo apt-get install -y -q libxml2-dev libxslt1-dev libxmlsec1-dev libxmlsec1-openssl opensc softhsm2 libengine-pkcs11-openssl + pip install --upgrade -r requirements-test.txt --no-binary lxml + pip install dist/xmlsec-$(python setup.py --version).tar.gz + + - name: Run tests + run: pytest -v --color=yes + + - name: Upload sdist + uses: actions/upload-artifact@v4.3.1 + with: + name: sdist + path: dist/*.tar.gz + + generate-wheels-matrix: + # Create a matrix of all architectures & versions to build. + # This enables the next step to run cibuildwheel in parallel. + # From https://iscinumpy.dev/post/cibuildwheel-2-10-0/#only-210 + name: Generate wheels matrix + runs-on: ubuntu-latest + outputs: + include: ${{ steps.set-matrix.outputs.include }} + steps: + - uses: actions/checkout@v4 + - name: Install cibuildwheel + # Nb. keep cibuildwheel version pin consistent with job below + run: pipx install cibuildwheel==2.16.5 + - id: set-matrix + # Once we have the windows build figured out, it can be added here + # by updating the matrix to include windows builds as well. + # See example here: + # https://github.com/lxml/lxml/blob/3ccc7d583e325ceb0ebdf8fc295bbb7fc8cd404d/.github/workflows/wheels.yml#L95C1-L106C51 + run: | + MATRIX=$( + { + cibuildwheel --print-build-identifiers --platform linux \ + | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \ + && cibuildwheel --print-build-identifiers --platform macos \ + | jq -nRc '{"only": inputs, "os": "macos-latest"}' \ + && cibuildwheel --print-build-identifiers --platform windows \ + | jq -nRc '{"only": inputs, "os": "windows-2019"}' + } | jq -sc + ) + echo "include=$MATRIX" + echo "include=$MATRIX" >> $GITHUB_OUTPUT + + build_wheels: + name: Build for ${{ matrix.only }} + needs: generate-wheels-matrix + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.generate-wheels-matrix.outputs.include) }} + + steps: + - name: Check out the repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v3 + with: + platforms: all + + - name: Build wheels + uses: pypa/cibuildwheel@v2.16.5 + with: + only: ${{ matrix.only }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/upload-artifact@v4.3.1 + with: + path: ./wheelhouse/*.whl + name: xmlsec-wheel-${{ matrix.only }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3080b068..820778ef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,16 +1,15 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks -exclude: ".*.diff" # exclude patches repos: - repo: https://github.com/psf/black - rev: 22.6.0 + rev: 24.3.0 hooks: - id: black types: [] files: ^.*.pyi?$ exclude: ^doc/ - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.6.0 hooks: - id: no-commit-to-branch - id: trailing-whitespace @@ -21,21 +20,22 @@ repos: - id: check-merge-conflict - id: check-json - id: detect-private-key + exclude: ^.*/rsakey.pem$ - id: mixed-line-ending - id: pretty-format-json args: [--autofix] - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 + rev: 7.0.0 hooks: - id: flake8 exclude: ^setup.py$ additional_dependencies: [flake8-docstrings, flake8-bugbear, flake8-logging-format, flake8-builtins, flake8-eradicate, flake8-fixme, pep8-naming, flake8-pep3101, flake8-annotations-complexity,flake8-pyi] - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.971 + rev: v1.9.0 hooks: - id: mypy exclude: (setup.py|tests/.*.py|doc/.*) @@ -43,6 +43,6 @@ repos: files: ^.*.pyi?$ additional_dependencies: [lxml-stubs,types-docutils] - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.9.0 + rev: v1.10.0 hooks: - id: rst-backticks diff --git a/.travis.yml b/.travis.yml index 9106805a..9e6ca540 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,9 @@ matrix: - python: 3.9 dist: xenial sudo: required + - python: 3.11 + dist: xenial + sudo: required env: global: - CFLAGS=-coverage diff --git a/README.rst b/README.rst index b2b3ab11..60fb3ea1 100644 --- a/README.rst +++ b/README.rst @@ -3,8 +3,9 @@ python-xmlsec .. image:: https://img.shields.io/pypi/v/xmlsec.svg?logo=python&logoColor=white :target: https://pypi.python.org/pypi/xmlsec -.. image:: https://img.shields.io/travis/com/mehcode/python-xmlsec/master.svg?logo=travis&logoColor=white&label=Travis%20CI - :target: https://travis-ci.org/mehcode/python-xmlsec +.. image:: https://results.pre-commit.ci/badge/github/xmlsec/python-xmlsec/master.svg + :target: https://results.pre-commit.ci/latest/github/xmlsec/python-xmlsec/master + :alt: pre-commit.ci status .. image:: https://img.shields.io/appveyor/ci/hoefling/xmlsec/master.svg?logo=appveyor&logoColor=white&label=AppVeyor :target: https://ci.appveyor.com/project/hoefling/xmlsec .. image:: https://github.com/mehcode/python-xmlsec/actions/workflows/manylinux.yml/badge.svg @@ -15,8 +16,8 @@ python-xmlsec :target: https://github.com/mehcode/python-xmlsec/actions/workflows/linuxbrew.yml .. image:: https://github.com/mehcode/python-xmlsec/actions/workflows/opensuse-tumbleweed.yml/badge.svg :target: https://github.com/mehcode/python-xmlsec/actions/workflows/opensuse-tumbleweed.yml -.. image:: https://codecov.io/gh/mehcode/python-xmlsec/branch/master/graph/badge.svg - :target: https://codecov.io/gh/mehcode/python-xmlsec +.. image:: https://codecov.io/gh/xmlsec/python-xmlsec/branch/master/graph/badge.svg + :target: https://codecov.io/gh/xmlsec/python-xmlsec .. image:: https://img.shields.io/readthedocs/xmlsec/latest?logo=read-the-docs :target: https://xmlsec.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status @@ -36,7 +37,7 @@ Check the `examples `_ se Requirements ************ - ``libxml2 >= 2.9.1`` -- ``libxmlsec1 >= 1.2.18`` +- ``libxmlsec1 >= 1.2.33`` Install ******* @@ -135,7 +136,7 @@ Building from source .. code-block:: bash - git clone https://github.com/mehcode/python-xmlsec.git + git clone https://github.com/xmlsec/python-xmlsec.git #. Change into the ``python-xmlsec`` root directory. diff --git a/doc/source/conf.py b/doc/source/conf.py index 7ff2b1f8..35329050 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -1,6 +1,6 @@ -# -*- coding: utf-8 -*- +from __future__ import annotations -import sys +import importlib.metadata import urllib.request import lxml @@ -11,12 +11,6 @@ from sphinx.environment import BuildEnvironment from sphinx.errors import ExtensionError -if sys.version_info >= (3, 8): - from importlib import metadata as importlib_metadata -else: - import importlib_metadata - - extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'sphinx.ext.intersphinx'] intersphinx_mapping = {'python': ('https://docs.python.org/3/', None)} @@ -28,11 +22,10 @@ project = u'python-xmlsec' copyright = u'2020, Oleg Hoefling ' # noqa: A001 author = u'Bulat Gaifullin ' -release = importlib_metadata.version('xmlsec') +release = importlib.metadata.version('xmlsec') parsed: Version = parse(release) version = '{}.{}'.format(parsed.major, parsed.minor) -language = None exclude_patterns: list[str] = [] pygments_style = 'sphinx' todo_include_todos = False @@ -69,6 +62,12 @@ autodoc_member_order = 'groupwise' autodoc_docstring_signature = True + +rst_prolog = ''' +.. role:: xml(code) + :language: xml +''' + # LXML crossref'ing stuff: # LXML doesn't have an intersphinx docs, # so we link to lxml.etree._Element explicitly diff --git a/doc/source/examples/encrypt.py b/doc/source/examples/encrypt.py index f69d4613..98f63b6f 100644 --- a/doc/source/examples/encrypt.py +++ b/doc/source/examples/encrypt.py @@ -2,10 +2,9 @@ import xmlsec -manager = xmlsec.KeysManager() -key = xmlsec.Key.from_file('rsacert.pem', xmlsec.constants.KeyDataFormatCertPem, None) -manager.add_key(key) -template = etree.parse('enc1-doc.xml').getroot() +with open('enc1-doc.xml') as fp: + template = etree.parse(fp).getroot() + enc_data = xmlsec.template.encrypted_data_create( template, xmlsec.constants.TransformAes128Cbc, @@ -20,6 +19,10 @@ data = template.find('./Data') # Encryption +manager = xmlsec.KeysManager() +key = xmlsec.Key.from_file('rsacert.pem', xmlsec.constants.KeyDataFormatCertPem, None) +manager.add_key(key) + enc_ctx = xmlsec.EncryptionContext(manager) enc_ctx.key = xmlsec.Key.generate( xmlsec.constants.KeyDataAes, 128, xmlsec.constants.KeyDataTypeSession diff --git a/doc/source/examples/sign.py b/doc/source/examples/sign.py index 4529bc8a..519c13a0 100644 --- a/doc/source/examples/sign.py +++ b/doc/source/examples/sign.py @@ -2,7 +2,8 @@ import xmlsec -template = etree.parse('sign1-tmpl.xml').getroot() +with open('sign1-tmpl.xml') as fp: + template = etree.parse(fp).getroot() signature_node = xmlsec.tree.find_node(template, xmlsec.constants.NodeSignature) ctx = xmlsec.SignatureContext() diff --git a/doc/source/examples/sign_binary.py b/doc/source/examples/sign_binary.py index 4e6c0e00..275c6e40 100644 --- a/doc/source/examples/sign_binary.py +++ b/doc/source/examples/sign_binary.py @@ -1,5 +1,3 @@ -from lxml import etree - import xmlsec ctx = xmlsec.SignatureContext() diff --git a/doc/source/examples/verify.py b/doc/source/examples/verify.py index 8629c550..808a53c2 100644 --- a/doc/source/examples/verify.py +++ b/doc/source/examples/verify.py @@ -2,7 +2,9 @@ import xmlsec -template = etree.parse('sign1-res.xml').getroot() +with open('sign1-res.xml') as fp: + template = etree.parse(fp).getroot() + xmlsec.tree.add_ids(template, ["ID"]) signature_node = xmlsec.tree.find_node(template, xmlsec.constants.NodeSignature) # Create a digital signature context (no key manager is needed). diff --git a/doc/source/examples/verify_binary.py b/doc/source/examples/verify_binary.py index 06c2b727..1f888b99 100644 --- a/doc/source/examples/verify_binary.py +++ b/doc/source/examples/verify_binary.py @@ -1,5 +1,3 @@ -from lxml import etree - import xmlsec ctx = xmlsec.SignatureContext() diff --git a/doc/source/index.rst b/doc/source/index.rst index 5cc758b9..e08f47d9 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -3,9 +3,6 @@ You can adapt this file completely to your liking, but it should at least contain the root ``toctree`` directive. -.. role:: xml(code) - :language: xml - Welcome to python-xmlsec's documentation! ========================================= diff --git a/doc/source/modules/constants.rst b/doc/source/modules/constants.rst index 4a63fcd7..3df6b50f 100644 --- a/doc/source/modules/constants.rst +++ b/doc/source/modules/constants.rst @@ -49,7 +49,11 @@ KeyData .. data:: xmlsec.constants.KeyDataEcdsa - The ECDSA key klass. + (Deprecated. The EC key klass) The ECDSA key klass. + +.. data:: xmlsec.constants.KeyDataEc + + The EC key klass. .. data:: xmlsec.constants.KeyDataHmac @@ -166,12 +170,6 @@ Namespaces .. data:: xmlsec.constants.XPointerNs :annotation: = 'http://www.w3.org/2001/04/xmldsig-more/xptr' -.. data:: xmlsec.constants.Soap11Ns - :annotation: = 'http://schemas.xmlsoap.org/soap/envelope/' - -.. data:: xmlsec.constants.Soap12Ns - :annotation: = 'http://www.w3.org/2002/06/soap-envelope' - .. data:: xmlsec.constants.NsExcC14N :annotation: = 'http://www.w3.org/2001/10/xml-exc-c14n#' diff --git a/doc/source/sphinx-pr-6916.diff b/doc/source/sphinx-pr-6916.diff deleted file mode 100644 index e7040a0f..00000000 --- a/doc/source/sphinx-pr-6916.diff +++ /dev/null @@ -1,46 +0,0 @@ -diff --git a/sphinx/environment/__init__.py b/sphinx/environment/__init__.py -index bc9bf49a74..4804c89c52 100644 ---- a/sphinx/environment/__init__.py -+++ b/sphinx/environment/__init__.py -@@ -46,6 +46,7 @@ - default_settings = { - 'embed_stylesheet': False, - 'cloak_email_addresses': True, -+ 'syntax_highlight': 'short', - 'pep_base_url': 'https://www.python.org/dev/peps/', - 'pep_references': None, - 'rfc_base_url': 'https://tools.ietf.org/html/', -diff --git a/sphinx/writers/html.py b/sphinx/writers/html.py -index 85eeb43..80f1eea 100644 ---- a/sphinx/writers/html.py -+++ b/sphinx/writers/html.py -@@ -494,8 +494,11 @@ class HTMLTranslator(SphinxTranslator, BaseTranslator): - self.body.append(self.starttag(node, 'kbd', '', - CLASS='docutils literal notranslate')) - else: -+ classes = 'docutils literal notranslate' -+ if 'code' in node['classes']: -+ classes += ' highlight' - self.body.append(self.starttag(node, 'code', '', -- CLASS='docutils literal notranslate')) -+ CLASS=classes)) - self.protect_literal_text += 1 - - def depart_literal(self, node: Element) -> None: -diff --git a/sphinx/writers/html5.py b/sphinx/writers/html5.py -index 80cedd3..470f559 100644 ---- a/sphinx/writers/html5.py -+++ b/sphinx/writers/html5.py -@@ -446,8 +446,11 @@ class HTML5Translator(SphinxTranslator, BaseTranslator): - self.body.append(self.starttag(node, 'kbd', '', - CLASS='docutils literal notranslate')) - else: -+ classes = 'docutils literal notranslate' -+ if 'code' in node['classes']: -+ classes += ' highlight' - self.body.append(self.starttag(node, 'code', '', -- CLASS='docutils literal notranslate')) -+ CLASS=classes)) - self.protect_literal_text += 1 - - def depart_literal(self, node: Element) -> None: diff --git a/pyproject.toml b/pyproject.toml index e28878e3..9b6469d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,3 +45,43 @@ known_third_party = ['lxml', 'pytest', '_pytest', 'hypothesis'] [build-system] requires = ['setuptools>=42', 'wheel', 'setuptools_scm[toml]>=3.4', "pkgconfig>=1.5.1", "lxml>=3.8, !=4.7.0"] + +[tool.cibuildwheel] +build-verbosity = 1 +build-frontend = "build" +skip = [ + "pp*", + "*-musllinux_i686", + # LXML doesn't publish wheels for these platforms, which makes it + # difficult for us to build wheels, so we exclude them. + "cp36-manylinux_aarch64", + "cp37-manylinux_aarch64", + "cp36-musllinux_aarch64", + "cp37-musllinux_aarch64", +] +test-command = "pytest -v --color=yes {package}/tests" +before-test = "pip install -r requirements-test.txt" +test-skip = "*-macosx_arm64" + +[tool.cibuildwheel.environment] +PYXMLSEC_STATIC_DEPS = "true" + +[tool.cibuildwheel.linux] +archs = ["x86_64", "aarch64", "i686"] +environment-pass = [ + "PYXMLSEC_LIBXML2_VERSION", + "PYXMLSEC_LIBXSLT_VERSION", + "PYXMLSEC_STATIC_DEPS", + "GH_TOKEN" +] + +[tool.cibuildwheel.macos] +archs = ["x86_64", "arm64"] +before-all = "brew install perl" + +[tool.cibuildwheel.windows] +archs = ["AMD64", "x86"] + +[[tool.cibuildwheel.overrides]] +select = "*-manylinux*" +before-all = "yum install -y perl-core" diff --git a/setup.cfg b/setup.cfg index 88d815b0..c090b4e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,5 +18,8 @@ upload_dir = doc/build/html [flake8] per-file-ignores = *.pyi: E301, E302, E305, E501, E701, F401, F822 + doc/source/conf.py: D1 + doc/source/examples/*.py: D1, E501 + tests/*.py: D1 exclude = .venv*,.git,*_pb2.pyi,build,dist,libs,.eggs,.direnv* max-line-length = 130 diff --git a/setup.py b/setup.py index 9a3c9277..92588ebc 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,8 @@ from distutils.errors import DistutilsError from distutils.version import StrictVersion as Version from pathlib import Path -from urllib.request import urlcleanup, urljoin, urlopen, urlretrieve +from urllib.parse import urljoin +from urllib.request import Request, urlcleanup, urlopen, urlretrieve from setuptools import Extension, setup from setuptools.command.build_ext import build_ext as build_ext_orig @@ -31,31 +32,60 @@ def handle_starttag(self, tag, attrs): self.hrefs.append(value) -def latest_release_from_html(url, matcher): - with contextlib.closing(urlopen(url)) as r: +def make_request(url, github_token=None, json_response=False): + headers = {'User-Agent': 'https://github.com/xmlsec/python-xmlsec'} + if github_token: + headers['authorization'] = "Bearer " + github_token + request = Request(url, headers=headers) + with contextlib.closing(urlopen(request)) as r: charset = r.headers.get_content_charset() or 'utf-8' content = r.read().decode(charset) - collector = HrefCollector() - collector.feed(content) - hrefs = collector.hrefs + if json_response: + return json.loads(content) + else: + return content - def comp(text): - try: - return Version(matcher.match(text).groupdict()['version']) - except (AttributeError, ValueError): - return Version('0.0') - latest = max(hrefs, key=comp) - return '{}/{}'.format(url, latest) +def latest_release_from_html(url, matcher): + content = make_request(url) + collector = HrefCollector() + collector.feed(content) + hrefs = collector.hrefs + + def comp(text): + try: + return Version(matcher.match(text).groupdict()['version']) + except (AttributeError, ValueError): + return Version('0.0') + + latest = max(hrefs, key=comp) + return '{}/{}'.format(url, latest) def latest_release_from_gnome_org_cache(url, lib_name): cache_url = '{}/cache.json'.format(url) - with contextlib.closing(urlopen(cache_url)) as r: - cache = json.load(r) - latest_version = cache[2][lib_name][-1] - latest_source = cache[1][lib_name][latest_version]['tar.xz'] - return '{}/{}'.format(url, latest_source) + cache = make_request(cache_url, json_response=True) + latest_version = cache[2][lib_name][-1] + latest_source = cache[1][lib_name][latest_version]['tar.xz'] + return '{}/{}'.format(url, latest_source) + + +def latest_release_from_github_api(repo): + api_url = 'https://api.github.com/repos/{}/releases'.format(repo) + + # if we are running in CI, pass along the GH_TOKEN, so we don't get rate limited + token = os.environ.get("GH_TOKEN") + if token: + log.info("Using GitHub token to avoid rate limiting") + api_releases = make_request(api_url, token, json_response=True) + releases = [r['tarball_url'] for r in api_releases if r['prerelease'] is False and r['draft'] is False] + if not releases: + raise DistutilsError('No release found for {}'.format(repo)) + return releases[0] + + +def latest_openssl_release(): + return latest_release_from_github_api('openssl/openssl') def latest_zlib_release(): @@ -78,6 +108,17 @@ def latest_xmlsec_release(): return latest_release_from_html('https://www.aleksey.com/xmlsec/download/', re.compile('xmlsec1-(?P.*).tar.gz')) +class CrossCompileInfo: + def __init__(self, host, arch, compiler): + self.host = host + self.arch = arch + self.compiler = compiler + + @property + def triplet(self): + return "{}-{}-{}".format(self.host, self.arch, self.compiler) + + class build_ext(build_ext_orig): def info(self, message): self.announce(message, level=log.INFO) @@ -86,6 +127,7 @@ def run(self): ext = self.ext_map['xmlsec'] self.debug = os.environ.get('PYXMLSEC_ENABLE_DEBUG', False) self.static = os.environ.get('PYXMLSEC_STATIC_DEPS', False) + self.size_opt = os.environ.get('PYXMLSEC_OPTIMIZE_SIZE', True) if self.static or sys.platform == 'win32': self.info('starting static build on {}'.format(sys.platform)) @@ -105,7 +147,9 @@ def run(self): if sys.platform == 'win32': self.prepare_static_build_win() elif 'linux' in sys.platform: - self.prepare_static_build_linux() + self.prepare_static_build(sys.platform) + elif 'darwin' in sys.platform: + self.prepare_static_build(sys.platform) else: import pkgconfig @@ -132,7 +176,7 @@ def run(self): [('MODULE_NAME', self.distribution.metadata.name), ('MODULE_VERSION', self.distribution.metadata.version)] ) # escape the XMLSEC_CRYPTO macro value, see mehcode/python-xmlsec#141 - for (key, value) in ext.define_macros: + for key, value in ext.define_macros: if key == 'XMLSEC_CRYPTO' and not (value.startswith('"') and value.endswith('"')): ext.define_macros.remove((key, value)) ext.define_macros.append((key, '"{0}"'.format(value))) @@ -153,28 +197,35 @@ def run(self): ) if self.debug: - ext.extra_compile_args.append('-Wall') - ext.extra_compile_args.append('-O0') ext.define_macros.append(('PYXMLSEC_ENABLE_DEBUG', '1')) + if sys.platform == 'win32': + ext.extra_compile_args.append('/Od') + else: + ext.extra_compile_args.append('-Wall') + ext.extra_compile_args.append('-O0') else: - ext.extra_compile_args.append('-Os') + if self.size_opt: + if sys.platform == 'win32': + ext.extra_compile_args.append('/Os') + else: + ext.extra_compile_args.append('-Os') super(build_ext, self).run() def prepare_static_build_win(self): - release_url = 'https://github.com/bgaifullin/libxml2-win-binaries/releases/download/v2018.08/' - if sys.maxsize > 2147483647: + release_url = 'https://github.com/mxamin/python-xmlsec-win-binaries/releases/download/2024.04.17/' + if sys.maxsize > 2147483647: # 2.0 GiB suffix = 'win64' else: suffix = 'win32' libs = [ - 'libxml2-2.9.4.{}.zip'.format(suffix), - 'libxslt-1.1.29.{}.zip'.format(suffix), - 'zlib-1.2.8.{}.zip'.format(suffix), - 'iconv-1.14.{}.zip'.format(suffix), - 'openssl-1.0.1.{}.zip'.format(suffix), - 'xmlsec-1.2.24.{}.zip'.format(suffix), + 'libxml2-2.11.7.{}.zip'.format(suffix), + 'libxslt-1.1.37.{}.zip'.format(suffix), + 'zlib-1.2.12.{}.zip'.format(suffix), + 'iconv-1.16-1.{}.zip'.format(suffix), + 'openssl-3.0.8.{}.zip'.format(suffix), + 'xmlsec-1.3.4.{}.zip'.format(suffix), ] for libfile in libs: @@ -211,7 +262,7 @@ def prepare_static_build_win(self): ext.libraries = [ 'libxmlsec_a', 'libxmlsec-openssl_a', - 'libeay32', + 'libcrypto', 'iconv_a', 'libxslt_a', 'libexslt_a', @@ -229,8 +280,8 @@ def prepare_static_build_win(self): includes.append(next(p / 'xmlsec' for p in includes if (p / 'xmlsec').is_dir())) ext.include_dirs = [str(p.absolute()) for p in includes] - def prepare_static_build_linux(self): - self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION', '1.1.1q') + def prepare_static_build(self, build_platform): + self.openssl_version = os.environ.get('PYXMLSEC_OPENSSL_VERSION') self.libiconv_version = os.environ.get('PYXMLSEC_LIBICONV_VERSION') self.libxml2_version = os.environ.get('PYXMLSEC_LIBXML2_VERSION') self.libxslt_version = os.environ.get('PYXMLSEC_LIBXSLT_VERSION') @@ -242,8 +293,13 @@ def prepare_static_build_linux(self): if openssl_tar is None: self.info('{:10}: {}'.format('OpenSSL', 'source tar not found, downloading ...')) openssl_tar = self.libs_dir / 'openssl.tar.gz' - self.info('{:10}: {} {}'.format('OpenSSL', 'version', self.openssl_version)) - urlretrieve('https://www.openssl.org/source/openssl-{}.tar.gz'.format(self.openssl_version), str(openssl_tar)) + if self.openssl_version is None: + url = latest_openssl_release() + self.info('{:10}: {}'.format('OpenSSL', 'PYXMLSEC_OPENSSL_VERSION unset, downloading latest from {}'.format(url))) + else: + url = 'https://api.github.com/repos/openssl/openssl/tarball/openssl-{}'.format(self.openssl_version) + self.info('{:10}: {} {}'.format('OpenSSL', 'version', self.openssl_version)) + urlretrieve(url, str(openssl_tar)) # fetch zlib zlib_tar = next(self.libs_dir.glob('zlib*.tar.gz'), None) @@ -344,16 +400,42 @@ def prepare_static_build_linux(self): prefix_arg = '--prefix={}'.format(self.prefix_dir) - cflags = ['-fPIC'] env = os.environ.copy() - if 'CFLAGS' in env: - env['CFLAGS'].append(' '.join(cflags)) - else: - env['CFLAGS'] = ' '.join(cflags) + cflags = [] + if env.get('CFLAGS'): + cflags.append(env['CFLAGS']) + cflags.append('-fPIC') + ldflags = [] + if env.get('LDFLAGS'): + ldflags.append(env['LDFLAGS']) + + cross_compiling = False + if build_platform == 'darwin': + import platform + + arch = self.plat_name.rsplit('-', 1)[1] + if arch != platform.machine() and arch in ('x86_64', 'arm64'): + self.info('Cross-compiling for {}'.format(arch)) + cflags.append('-arch {}'.format(arch)) + ldflags.append('-arch {}'.format(arch)) + cross_compiling = CrossCompileInfo('darwin64', arch, 'cc') + major_version, minor_version = tuple(map(int, platform.mac_ver()[0].split('.')[:2])) + if major_version >= 11: + if 'MACOSX_DEPLOYMENT_TARGET' not in env: + env['MACOSX_DEPLOYMENT_TARGET'] = "11.0" + + env['CFLAGS'] = ' '.join(cflags) + env['LDFLAGS'] = ' '.join(ldflags) self.info('Building OpenSSL') openssl_dir = next(self.build_libs_dir.glob('openssl-*')) - subprocess.check_output(['./config', prefix_arg, 'no-shared', '-fPIC'], cwd=str(openssl_dir), env=env) + openssl_config_cmd = [prefix_arg, 'no-shared', '-fPIC', '--libdir=lib'] + if cross_compiling: + openssl_config_cmd.insert(0, './Configure') + openssl_config_cmd.append(cross_compiling.triplet) + else: + openssl_config_cmd.insert(0, './config') + subprocess.check_output(openssl_config_cmd, cwd=str(openssl_dir), env=env) subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(openssl_dir), env=env) subprocess.check_output( ['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install_sw'], cwd=str(openssl_dir), env=env @@ -365,10 +447,22 @@ def prepare_static_build_linux(self): subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(zlib_dir), env=env) subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1), 'install'], cwd=str(zlib_dir), env=env) + host_arg = "" + if cross_compiling: + host_arg = '--host={}'.format(cross_compiling.arch) + self.info('Building libiconv') libiconv_dir = next(self.build_libs_dir.glob('libiconv-*')) subprocess.check_output( - ['./configure', prefix_arg, '--disable-dependency-tracking', '--disable-shared'], cwd=str(libiconv_dir), env=env + [ + './configure', + prefix_arg, + '--disable-dependency-tracking', + '--disable-shared', + host_arg, + ], + cwd=str(libiconv_dir), + env=env, ) subprocess.check_output(['make', '-j{}'.format(multiprocessing.cpu_count() + 1)], cwd=str(libiconv_dir), env=env) subprocess.check_output( @@ -383,11 +477,11 @@ def prepare_static_build_linux(self): prefix_arg, '--disable-dependency-tracking', '--disable-shared', - '--enable-rebuild-docs=no', '--without-lzma', '--without-python', '--with-iconv={}'.format(self.prefix_dir), '--with-zlib={}'.format(self.prefix_dir), + host_arg, ], cwd=str(libxml2_dir), env=env, @@ -408,6 +502,7 @@ def prepare_static_build_linux(self): '--without-python', '--without-crypto', '--with-libxml-prefix={}'.format(self.prefix_dir), + host_arg, ], cwd=str(libxslt_dir), env=env, @@ -418,10 +513,8 @@ def prepare_static_build_linux(self): ) self.info('Building xmlsec1') - if 'LDFLAGS' in env: - env['LDFLAGS'].append(' -lpthread') - else: - env['LDFLAGS'] = '-lpthread' + ldflags.append('-lpthread') + env['LDFLAGS'] = ' '.join(ldflags) xmlsec1_dir = next(self.build_libs_dir.glob('xmlsec1-*')) subprocess.check_output( [ @@ -429,6 +522,7 @@ def prepare_static_build_linux(self): prefix_arg, '--disable-shared', '--disable-gost', + '--enable-md5', '--disable-crypto-dl', '--enable-static=yes', '--enable-shared=no', @@ -437,6 +531,7 @@ def prepare_static_build_linux(self): '--with-openssl={}'.format(self.prefix_dir), '--with-libxml={}'.format(self.prefix_dir), '--with-libxslt={}'.format(self.prefix_dir), + host_arg, ], cwd=str(xmlsec1_dir), env=env, @@ -474,7 +569,8 @@ def prepare_static_build_linux(self): ext.include_dirs.extend([str(p.absolute()) for p in (self.prefix_dir / 'include').iterdir() if p.is_dir()]) ext.library_dirs = [] - ext.libraries = ['m', 'rt'] + if build_platform == 'linux': + ext.libraries = ['m', 'rt'] extra_objects = [ 'libxmlsec1.a', 'libxslt.a', @@ -503,6 +599,7 @@ def prepare_static_build_linux(self): use_scm_version=True, description='Python bindings for the XML Security Library', long_description=long_desc, + long_description_content_type='text/markdown', ext_modules=[pyxmlsec], cmdclass={'build_ext': build_ext}, python_requires='>=3.5', @@ -533,6 +630,7 @@ def prepare_static_build_linux(self): 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.11', 'Topic :: Text Processing :: Markup :: XML', 'Typing :: Typed', ], diff --git a/src/common.h b/src/common.h index 243ed651..a6176551 100644 --- a/src/common.h +++ b/src/common.h @@ -13,7 +13,7 @@ #include "debug.h" #ifndef MODULE_NAME -#define MODULE_NAME "xmlsec" +#define MODULE_NAME xmlsec #endif #define JOIN(X,Y) DO_JOIN1(X,Y) diff --git a/src/constants.c b/src/constants.c index 34c81b29..bd1fa5e0 100644 --- a/src/constants.c +++ b/src/constants.c @@ -316,8 +316,6 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_NS_CONSTANT(XPathNs, "XPATH"); PYXMLSEC_ADD_NS_CONSTANT(XPath2Ns, "XPATH2"); PYXMLSEC_ADD_NS_CONSTANT(XPointerNs, "XPOINTER"); - PYXMLSEC_ADD_NS_CONSTANT(Soap11Ns, "SOAP11"); - PYXMLSEC_ADD_NS_CONSTANT(Soap12Ns, "SOAP12"); PYXMLSEC_ADD_NS_CONSTANT(NsExcC14N, "EXC_C14N"); PYXMLSEC_ADD_NS_CONSTANT(NsExcC14NWithComments, "EXC_C14N_WITH_COMMENT"); @@ -441,13 +439,18 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataRetrievalMethod, "RETRIEVALMETHOD") PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataEncryptedKey, "ENCRYPTEDKEY") PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataAes, "AES") +#ifndef XMLSEC_NO_DES PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataDes, "DES") +#endif #ifndef XMLSEC_NO_DSA PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataDsa, "DSA") #endif -#if XMLSEC_VERSION_HEX > 0x10212 - // from version 1.2.19 +#if XMLSEC_VERSION_HEX > 0x10212 && XMLSEC_VERSION_HEX < 0x10303 + // from version 1.2.19 to version 1.3.2 (inclusive) PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataEcdsa, "ECDSA") +#elif XMLSEC_VERSION_HEX >= 0x10303 + // from version 1.3.3 (inclusive) + PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataEc, "ECDSA") #endif PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataHmac, "HMAC") PYXMLSEC_ADD_KEYDATA_CONSTANT(KeyDataRsa, "RSA") @@ -489,8 +492,10 @@ int PyXmlSec_ConstantsModule_Init(PyObject* package) { PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformKWAes192, "KW_AES192"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformKWAes256, "KW_AES256"); +#ifndef XMLSEC_NO_DES PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformDes3Cbc, "DES3"); PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformKWDes3, "KW_DES3"); +#endif #ifndef XMLSEC_NO_DSA PYXMLSEC_ADD_TRANSFORM_CONSTANT(TransformDsaSha1, "DSA_SHA1"); #endif diff --git a/src/enc.c b/src/enc.c index aaf35ae5..5453ef99 100644 --- a/src/enc.c +++ b/src/enc.c @@ -17,6 +17,11 @@ #include #include +// Backwards compatibility with xmlsec 1.2 +#ifndef XMLSEC_KEYINFO_FLAGS_LAX_KEY_SEARCH +#define XMLSEC_KEYINFO_FLAGS_LAX_KEY_SEARCH 0x00008000 +#endif + typedef struct { PyObject_HEAD xmlSecEncCtxPtr handle; @@ -50,6 +55,13 @@ static int PyXmlSec_EncryptionContext__init__(PyObject* self, PyObject* args, Py } ctx->manager = manager; PYXMLSEC_DEBUGF("%p: init enc context - ok, manager - %p", self, manager); + + // xmlsec 1.3 changed the key search to strict mode, causing various examples + // in the docs to fail. For backwards compatibility, this changes it back to + // lax mode for now. + ctx->handle->keyInfoReadCtx.flags = XMLSEC_KEYINFO_FLAGS_LAX_KEY_SEARCH; + ctx->handle->keyInfoWriteCtx.flags = XMLSEC_KEYINFO_FLAGS_LAX_KEY_SEARCH; + return 0; ON_FAIL: PYXMLSEC_DEBUGF("%p: init enc context - failed", self); diff --git a/src/keys.c b/src/keys.c index 1362b128..5ff04aae 100644 --- a/src/keys.c +++ b/src/keys.c @@ -163,7 +163,12 @@ static PyObject* PyXmlSec_KeyFromFile(PyObject* self, PyObject* args, PyObject* if (is_content) { key->handle = xmlSecCryptoAppKeyLoadMemory((const xmlSecByte*)data, (xmlSecSize)data_size, format, password, NULL, NULL); } else { - key->handle = xmlSecCryptoAppKeyLoad(data, format, password, NULL, NULL); + #if XMLSEC_VERSION_HEX >= 0x10303 + // from version 1.3.3 (inclusive) + key->handle = xmlSecCryptoAppKeyLoadEx(data, xmlSecKeyDataTypePrivate, format, password, NULL, NULL); + #else + key->handle = xmlSecCryptoAppKeyLoad(data, format, password, NULL, NULL); + #endif } Py_END_ALLOW_THREADS; @@ -185,6 +190,51 @@ static PyObject* PyXmlSec_KeyFromFile(PyObject* self, PyObject* args, PyObject* return NULL; } +static const char PyXmlSec_KeyFromEngine__doc__[] = \ + "from_engine(engine_and_key_id) -> xmlsec.Key\n" + "Loads PKI key from an engine.\n\n" + ":param engine_and_key_id: engine and key id, i.e. 'pkcs11;pkcs11:token=XmlsecToken;object=XmlsecKey;pin-value=password'\n" + ":type engine_and_key_id: :class:`str`, " + ":return: pointer to newly created key\n" + ":rtype: :class:`~xmlsec.Key`"; +static PyObject* PyXmlSec_KeyFromEngine(PyObject* self, PyObject* args, PyObject* kwargs) { + static char *kwlist[] = {"engine_and_key_id", NULL}; + + const char* engine_and_key_id = NULL; + PyXmlSec_Key* key = NULL; + + PYXMLSEC_DEBUG("load key from engine - start"); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s:from_engine", kwlist, &engine_and_key_id)) { + goto ON_FAIL; + } + + if ((key = PyXmlSec_NewKey1((PyTypeObject*)self)) == NULL) goto ON_FAIL; + + Py_BEGIN_ALLOW_THREADS; + #if XMLSEC_VERSION_HEX >= 0x10303 + // from version 1.3.3 (inclusive) + key->handle = xmlSecCryptoAppKeyLoadEx(engine_and_key_id, xmlSecKeyDataTypePrivate, xmlSecKeyDataFormatEngine, NULL, xmlSecCryptoAppGetDefaultPwdCallback(), (void*)engine_and_key_id); + #else + key->handle = xmlSecCryptoAppKeyLoad(engine_and_key_id, xmlSecKeyDataFormatEngine, NULL, xmlSecCryptoAppGetDefaultPwdCallback(), (void*)engine_and_key_id); + #endif + Py_END_ALLOW_THREADS; + + if (key->handle == NULL) { + PyXmlSec_SetLastError("cannot read key"); + goto ON_FAIL; + } + + key->is_own = 1; + + PYXMLSEC_DEBUG("load key from engine - ok"); + return (PyObject*)key; + +ON_FAIL: + PYXMLSEC_DEBUG("load key from engine - fail"); + Py_XDECREF(key); + return NULL; +} + static const char PyXmlSec_KeyGenerate__doc__[] = \ "generate(klass, size, type) -> xmlsec.Key\n" "Generates key of kind ``klass`` with ``size`` and ``type``.\n\n" @@ -494,6 +544,12 @@ static PyMethodDef PyXmlSec_KeyMethods[] = { METH_CLASS|METH_VARARGS|METH_KEYWORDS, PyXmlSec_KeyFromFile__doc__ }, + { + "from_engine", + (PyCFunction)PyXmlSec_KeyFromEngine, + METH_CLASS|METH_VARARGS|METH_KEYWORDS, + PyXmlSec_KeyFromEngine__doc__ + }, { "generate", (PyCFunction)PyXmlSec_KeyGenerate, diff --git a/src/lxml.c b/src/lxml.c index aa1abae0..c98e933b 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -9,6 +9,7 @@ #include "common.h" #include "lxml.h" +#include "exception.h" #include #include @@ -17,8 +18,95 @@ #include #include +#define XMLSEC_EXTRACT_VERSION(x, y) ((x / (y)) % 100) + +#define XMLSEC_EXTRACT_MAJOR(x) XMLSEC_EXTRACT_VERSION(x, 100 * 100) +#define XMLSEC_EXTRACT_MINOR(x) XMLSEC_EXTRACT_VERSION(x, 100) +#define XMLSEC_EXTRACT_PATCH(x) XMLSEC_EXTRACT_VERSION(x, 1) + +static long PyXmlSec_GetLibXmlVersionLong() { + return PyOS_strtol(xmlParserVersion, NULL, 10); +} +long PyXmlSec_GetLibXmlVersionMajor() { + return XMLSEC_EXTRACT_MAJOR(PyXmlSec_GetLibXmlVersionLong()); +} +long PyXmlSec_GetLibXmlVersionMinor() { + return XMLSEC_EXTRACT_MINOR(PyXmlSec_GetLibXmlVersionLong()); +} +long PyXmlSec_GetLibXmlVersionPatch() { + return XMLSEC_EXTRACT_PATCH(PyXmlSec_GetLibXmlVersionLong()); +} + +long PyXmlSec_GetLibXmlCompiledVersionMajor() { + return XMLSEC_EXTRACT_MAJOR(LIBXML_VERSION); +} +long PyXmlSec_GetLibXmlCompiledVersionMinor() { + return XMLSEC_EXTRACT_MINOR(LIBXML_VERSION); +} +long PyXmlSec_GetLibXmlCompiledVersionPatch() { + return XMLSEC_EXTRACT_PATCH(LIBXML_VERSION); +} + +static int PyXmlSec_CheckLxmlLibraryVersion(void) { + // Make sure that the version of libxml2 lxml is using is the same as the one we are using. Because + // we pass trees between the two libraries, we need to make sure that they are using the same version + // of libxml2, or we could run into difficult to debug segfaults. + // See: https://github.com/xmlsec/python-xmlsec/issues/283 + + PyObject* lxml = NULL; + PyObject* version = NULL; + + // Default to failure + int result = -1; + + lxml = PyImport_ImportModule("lxml.etree"); + if (lxml == NULL) { + goto FINALIZE; + } + version = PyObject_GetAttrString(lxml, "LIBXML_VERSION"); + if (version == NULL) { + goto FINALIZE; + } + if (!PyTuple_Check(version) || PyTuple_Size(version) < 2) { + goto FINALIZE; + } + + PyObject* major = PyTuple_GetItem(version, 0); + if (major == NULL) { + goto FINALIZE; + } + PyObject* minor = PyTuple_GetItem(version, 1); + if (minor == NULL) { + goto FINALIZE; + } + + if (!PyLong_Check(major) || !PyLong_Check(minor)) { + goto FINALIZE; + } + + if (PyLong_AsLong(major) != PyXmlSec_GetLibXmlVersionMajor() || PyLong_AsLong(minor) != PyXmlSec_GetLibXmlVersionMinor()) { + goto FINALIZE; + } + + result = 0; + +FINALIZE: + // Clear any errors that may have occurred + PyErr_Clear(); + + // Cleanup our references, and return the result + Py_XDECREF(lxml); + Py_XDECREF(version); + + return result; +} int PyXmlSec_InitLxmlModule(void) { + if (PyXmlSec_CheckLxmlLibraryVersion() < 0) { + PyXmlSec_SetLastError("lxml & xmlsec libxml2 library version mismatch"); + return -1; + } + return import_lxml__etree(); } diff --git a/src/lxml.h b/src/lxml.h index 6824076b..72050efe 100644 --- a/src/lxml.h +++ b/src/lxml.h @@ -29,4 +29,13 @@ PyXmlSec_LxmlElementPtr PyXmlSec_elementFactory(PyXmlSec_LxmlDocumentPtr doc, xm // converts o to PyObject, None object is not allowed, does not increment ref_counts int PyXmlSec_LxmlElementConverter(PyObject* o, PyXmlSec_LxmlElementPtr* p); +// get version numbers for libxml2 both compiled and loaded +long PyXmlSec_GetLibXmlVersionMajor(); +long PyXmlSec_GetLibXmlVersionMinor(); +long PyXmlSec_GetLibXmlVersionPatch(); + +long PyXmlSec_GetLibXmlCompiledVersionMajor(); +long PyXmlSec_GetLibXmlCompiledVersionMinor(); +long PyXmlSec_GetLibXmlCompiledVersionPatch(); + #endif // __PYXMLSEC_LXML_H__ diff --git a/src/main.c b/src/main.c index 41f5e5fc..61eac139 100644 --- a/src/main.c +++ b/src/main.c @@ -10,6 +10,7 @@ #include "common.h" #include "platform.h" #include "exception.h" +#include "lxml.h" #include #include @@ -119,6 +120,37 @@ static PyObject* PyXmlSec_PyShutdown(PyObject* self) { Py_RETURN_NONE; } +static char PyXmlSec_GetLibXmlSecVersion__doc__[] = \ + "get_libxmlsec_version() -> tuple\n" + "Returns Version tuple of wrapped libxmlsec library."; +static PyObject* PyXmlSec_GetLibXmlSecVersion() { + return Py_BuildValue("(iii)", XMLSEC_VERSION_MAJOR, XMLSEC_VERSION_MINOR, XMLSEC_VERSION_SUBMINOR); +} + +static char PyXmlSec_GetLibXmlVersion__doc__[] = \ + "get_libxml_version() -> tuple[int, int, int]\n" + "Returns version tuple of libxml2 library xmlsec is using."; +static PyObject* PyXmlSec_GetLibXmlVersion() { + return Py_BuildValue( + "(iii)", + PyXmlSec_GetLibXmlVersionMajor(), + PyXmlSec_GetLibXmlVersionMinor(), + PyXmlSec_GetLibXmlVersionPatch() + ); +} + +static char PyXmlSec_GetLibXmlCompiledVersion__doc__[] = \ + "get_libxml_compiled_version() -> tuple[int, int, int]\n" + "Returns version tuple of libxml2 library xmlsec was compiled with."; +static PyObject* PyXmlSec_GetLibXmlCompiledVersion() { + return Py_BuildValue( + "(iii)", + PyXmlSec_GetLibXmlCompiledVersionMajor(), + PyXmlSec_GetLibXmlCompiledVersionMinor(), + PyXmlSec_GetLibXmlCompiledVersionPatch() + ); +} + static char PyXmlSec_PyEnableDebugOutput__doc__[] = \ "enable_debug_trace(enabled) -> None\n" "Enables or disables calling LibXML2 callback from the default errors callback.\n\n" @@ -275,18 +307,23 @@ static PyObject* PyXmlSec_PyIORegisterDefaultCallbacks(PyObject *self) { } static char PyXmlSec_PyIORegisterCallbacks__doc__[] = \ + "register_callbacks(input_match_callback, input_open_callback, input_read_callback, input_close_callback) -> None\n" "Register globally a custom set of IO callbacks with xmlsec.\n\n" - ":param callable input_match_callback: A callable that takes a filename `bytestring` and " + ":param input_match_callback: A callable that takes a filename `bytestring` and " "returns a boolean as to whether the other callbacks in this set can handle that name.\n" - ":param callable input_open_callback: A callable that takes a filename and returns some " + ":type input_match_callback: ~collections.abc.Callable[[bytes], bool]\n" + ":param input_open_callback: A callable that takes a filename and returns some " "context object (e.g. a file object) that the remaining callables in this set will be passed " "during handling.\n" + ":type input_open_callback: ~collections.abc.Callable[[bytes], Any]\n" // FIXME: How do we handle failures in ^^ (e.g. can't find the file)? - ":param callable input_read_callback: A callable that that takes the context object from the " + ":param input_read_callback: A callable that that takes the context object from the " "open callback and a buffer, and should fill the buffer with data (e.g. BytesIO.readinto()). " "xmlsec will call this function several times until there is no more data returned.\n" - ":param callable input_close_callback: A callable that takes the context object from the " + ":type input_read_callback: ~collections.abc.Callable[[Any, memoryview], int]\n" + ":param input_close_callback: A callable that takes the context object from the " "open callback and can do any resource cleanup necessary.\n" + ":type input_close_callback: ~collections.abc.Callable[[Any], None]\n" ; static PyObject* PyXmlSec_PyIORegisterCallbacks(PyObject *self, PyObject *args, PyObject *kwargs) { static char *kwlist[] = { @@ -381,6 +418,24 @@ static PyMethodDef PyXmlSec_MainMethods[] = { METH_NOARGS, PyXmlSec_PyShutdown__doc__ }, + { + "get_libxmlsec_version", + (PyCFunction)PyXmlSec_GetLibXmlSecVersion, + METH_NOARGS, + PyXmlSec_GetLibXmlSecVersion__doc__ + }, + { + "get_libxml_version", + (PyCFunction)PyXmlSec_GetLibXmlVersion, + METH_NOARGS, + PyXmlSec_GetLibXmlVersion__doc__ + }, + { + "get_libxml_compiled_version", + (PyCFunction)PyXmlSec_GetLibXmlCompiledVersion, + METH_NOARGS, + PyXmlSec_GetLibXmlCompiledVersion__doc__ + }, { "enable_debug_trace", (PyCFunction)PyXmlSec_PyEnableDebugOutput, diff --git a/src/xmlsec/__init__.pyi b/src/xmlsec/__init__.pyi index be0bd659..9cfc8cc6 100644 --- a/src/xmlsec/__init__.pyi +++ b/src/xmlsec/__init__.pyi @@ -5,6 +5,7 @@ from _typeshed import GenericPath, Self, StrOrBytesPath from lxml.etree import _Element from xmlsec import constants as constants +from xmlsec import template as template from xmlsec import tree as tree from xmlsec.constants import __KeyData as KeyData from xmlsec.constants import __Transform as Transform @@ -12,6 +13,8 @@ from xmlsec.constants import __Transform as Transform _E = TypeVar('_E', bound=_Element) def enable_debug_trace(enabled: bool = ...) -> None: ... +def get_libxml_version() -> tuple[int, int, int]: ... +def get_libxml_compiled_version() -> tuple[int, int, int]: ... def init() -> None: ... def shutdown() -> None: ... def cleanup_callbacks() -> None: ... @@ -48,6 +51,8 @@ class Key: @classmethod def from_file(cls: type[Self], file: GenericPath[AnyStr] | IO[AnyStr], format: int, password: str | None = ...) -> Self: ... @classmethod + def from_engine(cls: type[Self], engine_and_key_id: AnyStr) -> Self: ... + @classmethod def from_memory(cls: type[Self], data: AnyStr, format: int, password: str | None = ...) -> Self: ... @classmethod def generate(cls: type[Self], klass: KeyData, size: int, type: int) -> Self: ... diff --git a/src/xmlsec/constants.pyi b/src/xmlsec/constants.pyi index 3430a027..3c3ea94f 100644 --- a/src/xmlsec/constants.pyi +++ b/src/xmlsec/constants.pyi @@ -29,6 +29,7 @@ EncNs: Final[str] KeyDataAes: Final[__KeyData] KeyDataDes: Final[__KeyData] KeyDataDsa: Final[__KeyData] +KeyDataEc: Final[__KeyData] KeyDataEcdsa: Final[__KeyData] KeyDataEncryptedKey: Final[__KeyData] KeyDataFormatBinary: Final[int] @@ -85,8 +86,6 @@ NodeX509Data: Final[str] Ns: Final[str] NsExcC14N: Final[str] NsExcC14NWithComments: Final[str] -Soap11Ns: Final[str] -Soap12Ns: Final[str] TransformAes128Cbc: Final[__Transform] TransformAes128Gcm: Final[__Transform] TransformAes192Cbc: Final[__Transform] diff --git a/tests/base.py b/tests/base.py index 06acf413..48aef81d 100644 --- a/tests/base.py +++ b/tests/base.py @@ -7,7 +7,7 @@ import xmlsec -etype = type(etree.Element("test")) +etype = type(etree.Element('test')) ns = {'dsig': xmlsec.constants.DSigNs, 'enc': xmlsec.constants.EncNs} @@ -15,30 +15,15 @@ try: import resource - def get_memory_usage(): - return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - -except ImportError: - resource = None - - def get_memory_usage(): - return 0 - - -def get_iterations(): - if sys.platform in ('win32',): - return 0 - - try: - return int(os.getenv("PYXMLSEC_TEST_ITERATIONS", "10")) - except ValueError: - return 0 + test_iterations = int(os.environ.get('PYXMLSEC_TEST_ITERATIONS', '10')) +except (ImportError, ValueError): + test_iterations = 0 class TestMemoryLeaks(unittest.TestCase): maxDiff = None - iterations = get_iterations() + iterations = test_iterations data_dir = os.path.join(os.path.dirname(__file__), "data") @@ -114,9 +99,9 @@ def assertXmlEqual(self, first, second, msg=None): # noqa: N802 for name in second.attrib.keys(): if name not in first.attrib: self.fail('x2 has an attribute x1 is missing: {}. {}'.format(name, msg)) - if not xml_text_compare(first.text, second.text): + if not _xml_text_compare(first.text, second.text): self.fail('text: {!r} != {!r}. {}'.format(first.text, second.text, msg)) - if not xml_text_compare(first.tail, second.tail): + if not _xml_text_compare(first.tail, second.tail): self.fail('tail: {!r} != {!r}. {}'.format(first.tail, second.tail, msg)) cl1 = sorted(first.getchildren(), key=lambda x: x.tag) cl2 = sorted(second.getchildren(), key=lambda x: x.tag) @@ -128,7 +113,7 @@ def assertXmlEqual(self, first, second, msg=None): # noqa: N802 self.assertXmlEqual(c1, c2) -def xml_text_compare(t1, t2): +def _xml_text_compare(t1, t2): if not t1 and not t2: return True if t1 == '*' or t2 == '*': diff --git a/tests/conftest.py b/tests/conftest.py index b51c4d45..675258c5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,8 @@ def pytest_collection_modifyitems(items): """ - Put the module init test first to implicitly check whether - any subsequent test fails because of module reinitialization. + Put the module init test first. + + This way, we implicitly check whether any subsequent test fails because of module reinitialization. """ def module_init_tests_first(item): diff --git a/tests/data/enc-bad-in.xml b/tests/data/enc-bad-in.xml deleted file mode 100644 index 460738fc..00000000 --- a/tests/data/enc-bad-in.xml +++ /dev/null @@ -1,208 +0,0 @@ - - - - - - - - - MyNextCar - CreditApplication - MYNEXTCAR - VW - 409D03 - MyNextCar - - 2018-11-20T09:37:45Z - 7f0842cc-8d47-4955-be31-c61d07ee490b - - VW - - - - - - -
- VCI_MNA_0000070250 - - - Car Chantilly -
- 14839 Stonecroft Center Ct - Chantilly - VA - US - 20151 -
- - - MyNextCar - MNA - - 7039562100 - - CAR - - N -
- - - 2017 - Q7 - CAR - New - 0 - Prestige - - 64300.0 - MSRP - - - 64300.0 - Selling Price - - - - - 113456789 - NationalId - - - John - Q - Public - -
- 999 Washington Ave - Apt #332 - Front Royal - VA - US - 22630 - 01 - 10 - Own -
-
- 21 E 9th Ave - Boulder - CO - US - 80301-7577 - 07 - 11 - Own -
- - 3032852402 - 3032852405 - 7203554444 - JohnQPublic@anydomain.org - - - 1967-07-31 - - 0 - - UPS -
- 1775 Wiehle Ave. - Reston - VA - US - 20190 -
- 9500.0 - Driver - 01 - 05 - Current - - - FedEx - 4000.00 - Driver - 04 - 09 - Previous - - 1252.52 - - 1500.00 - - - 1 - Consents to Credit Check - - - - - 123435325 - NationalId - - - Lisa - C - Public - -
- 999 Lewis Street - Front Royal - VA - US - 22630 - 5 - 0 - Own -
- - 5401110000 - 5401110073 - public@test.com - - - 1963-04-20 - - - Christendom College -
- 999 Christendom Dr - Front Royal - VA - US - 22630 -
- 6200.00 - Professor - 5 - 0 - Current - - 1252.52 - - 1 - Consents to Credit Check - - - - R - 0.00 - 66 - 5000.00 - INDIVCOAPP - 2000.00 - MyNextCar - - 1978 - Bonneville - Pontiac - Coupe - - - - - - - - diff --git a/tests/data/sign5-out-xmlsec_1_2_36_to_37.xml b/tests/data/sign5-out-xmlsec_1_2_36_to_37.xml new file mode 100644 index 00000000..f359b138 --- /dev/null +++ b/tests/data/sign5-out-xmlsec_1_2_36_to_37.xml @@ -0,0 +1,67 @@ + + + + + Hello, World! + + + + + + + + + + +HjY8ilZAIEM2tBbPn5mYO1ieIX4= + + +SIaj/6KY3C1SmDXU2++Gm31U1xTadFp04WhBgfsJFbxrL+q7GKSKN9kfQ+UpN9+i +D5fWmuavXEHe4Gw6RMaMEkq2URQo7F68+d5J/ajq8/l4n+xE6/reGScVwT6L4dEP +XXVJcAi2ZnQ3O7GTNvNGCPibL9mUcyCWBFZ92Uemtc/vJFCQ7ZyKMdMfACgxOwyN +T/9971oog241/2doudhonc0I/3mgPYWkZdX6yvr62mEjnG+oUZkhWYJ4ewZJ4hM4 +JjbFqZO+OEzDRSbw3DkmuBA/mtlx+3t13SESfEub5hqoMdVmtth/eTb64dsPdl9r +3k1ACVX9f8aHfQQdJOmLFQ== + + + + + + +Test Issuer +1 + +MIIE3zCCBEigAwIBAgIBBTANBgkqhkiG9w0BAQQFADCByzELMAkGA1UEBhMCVVMx +EzARBgNVBAgTCkNhbGlmb3JuaWExEjAQBgNVBAcTCVN1bm55dmFsZTE9MDsGA1UE +ChM0WE1MIFNlY3VyaXR5IExpYnJhcnkgKGh0dHA6Ly93d3cuYWxla3NleS5jb20v +eG1sc2VjKTEZMBcGA1UECxMQUm9vdCBDZXJ0aWZpY2F0ZTEWMBQGA1UEAxMNQWxl +a3NleSBTYW5pbjEhMB8GCSqGSIb3DQEJARYSeG1sc2VjQGFsZWtzZXkuY29tMB4X +DTAzMDMzMTA0MDIyMloXDTEzMDMyODA0MDIyMlowgb8xCzAJBgNVBAYTAlVTMRMw +EQYDVQQIEwpDYWxpZm9ybmlhMT0wOwYDVQQKEzRYTUwgU2VjdXJpdHkgTGlicmFy +eSAoaHR0cDovL3d3dy5hbGVrc2V5LmNvbS94bWxzZWMpMSEwHwYDVQQLExhFeGFt +cGxlcyBSU0EgQ2VydGlmaWNhdGUxFjAUBgNVBAMTDUFsZWtzZXkgU2FuaW4xITAf +BgkqhkiG9w0BCQEWEnhtbHNlY0BhbGVrc2V5LmNvbTCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBAJe4/rQ/gzV4FokE7CthjL/EXwCBSkXm2c3p4jyXO0Wt +quaNC3dxBwFPfPl94hmq3ZFZ9PHPPbp4RpYRnLZbRjlzVSOq954AXOXpSew7nD+E +mTqQrd9+ZIbGJnLOMQh5fhMVuOW/1lYCjWAhTCcYZPv7VXD2M70vVXDVXn6ZrqTg +qkVHE6gw1aCKncwg7OSOUclUxX8+Zi10v6N6+PPslFc5tKwAdWJhVLTQ4FKG+F53 +7FBDnNK6p4xiWryy/vPMYn4jYGvHUUk3eH4lFTCr+rSuJY8i/KNIf/IKim7g/o3w +Ae3GM8xrof2mgO8GjK/2QDqOQhQgYRIf4/wFsQXVZcMCAwEAAaOCAVcwggFTMAkG +A1UdEwQCMAAwLAYJYIZIAYb4QgENBB8WHU9wZW5TU0wgR2VuZXJhdGVkIENlcnRp +ZmljYXRlMB0GA1UdDgQWBBQkhCzy1FkgYosuXIaQo6owuicanDCB+AYDVR0jBIHw +MIHtgBS0ue+a5pcOaGUemM76VQ2JBttMfKGB0aSBzjCByzELMAkGA1UEBhMCVVMx +EzARBgNVBAgTCkNhbGlmb3JuaWExEjAQBgNVBAcTCVN1bm55dmFsZTE9MDsGA1UE +ChM0WE1MIFNlY3VyaXR5IExpYnJhcnkgKGh0dHA6Ly93d3cuYWxla3NleS5jb20v +eG1sc2VjKTEZMBcGA1UECxMQUm9vdCBDZXJ0aWZpY2F0ZTEWMBQGA1UEAxMNQWxl +a3NleSBTYW5pbjEhMB8GCSqGSIb3DQEJARYSeG1sc2VjQGFsZWtzZXkuY29tggEA +MA0GCSqGSIb3DQEBBAUAA4GBALU/mzIxSv8vhDuomxFcplzwdlLZbvSQrfoNkMGY +1UoS3YJrN+jZLWKSyWE3mIaPpElqXiXQGGkwD5iPQ1iJMbI7BeLvx6ZxX/f+c8Wn +ss0uc1NxfahMaBoyG15IL4+beqO182fosaKJTrJNG3mc//ANGU9OsQM9mfBEt4oL +NJ2D + + + + + diff --git a/tests/softhsm_setup.py b/tests/softhsm_setup.py new file mode 100644 index 00000000..247f1b18 --- /dev/null +++ b/tests/softhsm_setup.py @@ -0,0 +1,332 @@ +"""Testing the PKCS#11 shim layer. + +Heavily inspired by from https://github.com/IdentityPython/pyXMLSecurity by leifj +under license "As is", see https://github.com/IdentityPython/pyXMLSecurity/blob/master/LICENSE.txt +""" + +import logging +import os +import shutil +import subprocess +import tempfile +import traceback +import unittest + +DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') + + +def paths_for_component(component: str, default_paths): + env_path = os.environ.get(component) + return [env_path] if env_path else default_paths + + +def find_alts(component_name, alts) -> str: + for a in alts: + if os.path.exists(a): + return a + raise unittest.SkipTest('Required component is missing: {}'.format(component_name)) + + +def run_cmd(args, softhsm_conf=None): + env = {} + if softhsm_conf is not None: + env['SOFTHSM_CONF'] = softhsm_conf + env['SOFTHSM2_CONF'] = softhsm_conf + proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + out, err = proc.communicate() + if err is not None and len(err) > 0: + logging.error(err) + if out is not None and len(out) > 0: + logging.debug(out) + rv = proc.wait() + if rv: + with open(softhsm_conf) as f: + conf = f.read() + msg = '[cmd: {cmd}] [code: {code}] [stdout: {out}] [stderr: {err}] [config: {conf}]' + msg = msg.format( + cmd=' '.join(args), + code=rv, + out=out.strip(), + err=err.strip(), + conf=conf, + ) + raise RuntimeError(msg) + return out, err + + +component_default_paths = { + 'P11_MODULE': [ + '/usr/lib/softhsm/libsofthsm2.so', + '/usr/lib/x86_64-linux-gnu/softhsm/libsofthsm2.so', + '/usr/lib/softhsm/libsofthsm.so', + '/usr/lib64/softhsm/libsofthsm2.so', + ], + 'P11_ENGINE': [ + '/usr/lib/ssl/engines/libpkcs11.so', + '/usr/lib/engines/engine_pkcs11.so', + '/usr/lib/x86_64-linux-gnu/engines-1.1/pkcs11.so', + '/usr/lib64/engines-1.1/pkcs11.so', + '/usr/lib64/engines-1.1/libpkcs11.so', + '/usr/lib64/engines-3/pkcs11.so', + '/usr/lib64/engines-3/libpkcs11.so', + '/usr/lib/x86_64-linux-gnu/engines-3/pkcs11.so', + '/usr/lib/x86_64-linux-gnu/engines-3/libpkcs11.so', + ], + 'PKCS11_TOOL': [ + '/usr/bin/pkcs11-tool', + ], + 'SOFTHSM': [ + '/usr/bin/softhsm2-util', + '/usr/bin/softhsm', + ], + 'OPENSSL': [ + '/usr/bin/openssl', + ], +} + +component_path = { + component_name: find_alts(component_name, paths_for_component(component_name, default_paths)) + for component_name, default_paths in component_default_paths.items() +} + +softhsm_version = 1 +if component_path['SOFTHSM'].endswith('softhsm2-util'): + softhsm_version = 2 + +openssl_version = subprocess.check_output([component_path['OPENSSL'], 'version'])[8:11].decode() + +p11_test_files = [] +softhsm_conf = None +softhsm_db = None + + +def _temp_file() -> str: + f = tempfile.NamedTemporaryFile(delete=False) + p11_test_files.append(f.name) + return f.name + + +def _temp_dir() -> str: + d = tempfile.mkdtemp() + p11_test_files.append(d) + return d + + +@unittest.skipIf(component_path['P11_MODULE'] is None, 'SoftHSM PKCS11 module not installed') +def setup() -> None: + logging.debug('Creating test pkcs11 token using softhsm') + try: + global softhsm_conf + softhsm_conf = _temp_file() + logging.debug('Generating softhsm.conf') + with open(softhsm_conf, 'w') as f: + if softhsm_version == 2: + softhsm_db = _temp_dir() + f.write( + """ +# Generated by test +directories.tokendir = {} +objectstore.backend = file +log.level = DEBUG +""".format( + softhsm_db + ) + ) + else: + softhsm_db = _temp_file() + f.write( + """ +# Generated by test +0:{} +""".format( + softhsm_db + ) + ) + + logging.debug('Initializing the token') + out, err = run_cmd( + [ + component_path['SOFTHSM'], + '--slot', + '0', + '--label', + 'test', + '--init-token', + '--pin', + 'secret1', + '--so-pin', + 'secret2', + ], + softhsm_conf=softhsm_conf, + ) + + hash_priv_key = _temp_file() + logging.debug('Converting test private key to format for softhsm') + run_cmd( + [ + component_path['OPENSSL'], + 'pkcs8', + '-topk8', + '-inform', + 'PEM', + '-outform', + 'PEM', + '-nocrypt', + '-in', + os.path.join(DATA_DIR, 'rsakey.pem'), + '-out', + hash_priv_key, + ], + softhsm_conf=softhsm_conf, + ) + + logging.debug('Importing the test key to softhsm') + run_cmd( + [ + component_path['SOFTHSM'], + '--import', + hash_priv_key, + '--token', + 'test', + '--id', + 'a1b2', + '--label', + 'test', + '--pin', + 'secret1', + ], + softhsm_conf=softhsm_conf, + ) + run_cmd( + [ + component_path['PKCS11_TOOL'], + '--module', + component_path['P11_MODULE'], + '-l', + '--pin', + 'secret1', + '-O', + ], + softhsm_conf=softhsm_conf, + ) + signer_cert_pem = _temp_file() + openssl_conf = _temp_file() + logging.debug('Generating OpenSSL config for version %s', openssl_version) + with open(openssl_conf, 'w') as f: + f.write( + '\n'.join( + [ + 'openssl_conf = openssl_def', + '[openssl_def]', + 'engines = engine_section', + '[engine_section]', + 'pkcs11 = pkcs11_section', + '[req]', + 'distinguished_name = req_distinguished_name', + '[req_distinguished_name]', + '[pkcs11_section]', + 'engine_id = pkcs11', + # dynamic_path, + "MODULE_PATH = {}".format(component_path['P11_MODULE']), + 'init = 0', + ] + ) + ) + + with open(openssl_conf, 'r') as f: + logging.debug('-------- START DEBUG openssl_conf --------') + logging.debug(f.readlines()) + logging.debug('-------- END DEBUG openssl_conf --------') + logging.debug('-------- START DEBUG paths --------') + logging.debug(run_cmd(['ls', '-ld', component_path['P11_ENGINE']])) + logging.debug(run_cmd(['ls', '-ld', component_path['P11_MODULE']])) + logging.debug('-------- END DEBUG paths --------') + + signer_cert_der = _temp_file() + + logging.debug('Generating self-signed certificate') + run_cmd( + [ + component_path['OPENSSL'], + 'req', + '-new', + '-x509', + '-subj', + '/CN=Test Signer', + '-engine', + 'pkcs11', + '-config', + openssl_conf, + '-keyform', + 'engine', + '-key', + 'label_test', + '-passin', + 'pass:secret1', + '-out', + signer_cert_pem, + ], + softhsm_conf=softhsm_conf, + ) + + run_cmd( + [ + component_path['OPENSSL'], + 'x509', + '-inform', + 'PEM', + '-outform', + 'DER', + '-in', + signer_cert_pem, + '-out', + signer_cert_der, + ], + softhsm_conf=softhsm_conf, + ) + + logging.debug('Importing certificate into token') + + run_cmd( + [ + component_path['PKCS11_TOOL'], + '--module', + component_path['P11_MODULE'], + '-l', + '--slot-index', + '0', + '--id', + 'a1b2', + '--label', + 'test', + '-y', + 'cert', + '-w', + signer_cert_der, + '--pin', + 'secret1', + ], + softhsm_conf=softhsm_conf, + ) + + # TODO: Should be teardowned in teardown # noqa: T101 + os.environ['SOFTHSM_CONF'] = softhsm_conf + os.environ['SOFTHSM2_CONF'] = softhsm_conf + + except Exception as ex: + print('-' * 64) + traceback.print_exc() + print('-' * 64) + logging.exception('PKCS11 tests disabled: unable to initialize test token') + raise ex + + +def teardown() -> None: + global p11_test_files + for o in p11_test_files: + if os.path.exists(o): + if os.path.isdir(o): + shutil.rmtree(o) + else: + os.unlink(o) + p11_test_files = [] diff --git a/tests/test_ds.py b/tests/test_ds.py index 9417fedb..38f0b25c 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -132,7 +132,6 @@ def test_sign_case3(self): def test_sign_case4(self): """Should sign a file using a dynamically created template, key from PEM and an X509 cert with custom ns.""" - root = self.load_xml("sign4-in.xml") xmlsec.tree.add_ids(root, ["ID"]) elem_id = root.get('ID', None) @@ -183,7 +182,11 @@ def test_sign_case5(self): self.assertEqual("rsakey.pem", ctx.key.name) ctx.sign(sign) - self.assertEqual(self.load_xml("sign5-out.xml"), root) + if (1, 2, 36) <= xmlsec.get_libxmlsec_version() <= (1, 2, 37): + expected_xml_file = 'sign5-out-xmlsec_1_2_36_to_37.xml' + else: + expected_xml_file = 'sign5-out.xml' + self.assertEqual(self.load_xml(expected_xml_file), root) def test_sign_binary_bad_args(self): ctx = xmlsec.SignatureContext() @@ -249,7 +252,7 @@ def test_verify_case_5(self): self.check_verify(5) def check_verify(self, i): - root = self.load_xml("sign%d-out.xml" % i) + root = self.load_xml("sign{}-out.xml".format(i)) xmlsec.tree.add_ids(root, ["ID"]) sign = xmlsec.tree.find_node(root, consts.NodeSignature) self.assertIsNotNone(sign) diff --git a/tests/test_enc.py b/tests/test_enc.py index 7add848c..1788b4d6 100644 --- a/tests/test_enc.py +++ b/tests/test_enc.py @@ -1,4 +1,3 @@ -import os import tempfile from lxml import etree @@ -125,7 +124,7 @@ def test_encrypt_binary(self): encrypted = ctx.encrypt_binary(enc_data, b'test') self.assertIsNotNone(encrypted) - self.assertEqual("{%s}%s" % (consts.EncNs, consts.NodeEncryptedData), encrypted.tag) + self.assertEqual("{{{}}}{}".format(consts.EncNs, consts.NodeEncryptedData), encrypted.tag) enc_method = xmlsec.tree.find_child(enc_data, consts.NodeEncryptionMethod, consts.EncNs) self.assertIsNotNone(enc_method) @@ -170,7 +169,7 @@ def test_encrypt_uri(self): encrypted = ctx.encrypt_binary(enc_data, 'file://' + tmpfile.name) self.assertIsNotNone(encrypted) - self.assertEqual("{%s}%s" % (consts.EncNs, consts.NodeEncryptedData), encrypted.tag) + self.assertEqual("{{{}}}{}".format(consts.EncNs, consts.NodeEncryptedData), encrypted.tag) enc_method = xmlsec.tree.find_child(enc_data, consts.NodeEncryptionMethod, consts.EncNs) self.assertIsNotNone(enc_method) @@ -219,7 +218,7 @@ def test_decrypt_key(self): self.assertEqual(self.load_xml("enc3-in.xml"), decrypted) def check_decrypt(self, i): - root = self.load_xml('enc%d-out.xml' % i) + root = self.load_xml('enc{}-out.xml'.format(i)) enc_data = xmlsec.tree.find_child(root, consts.NodeEncryptedData, consts.EncNs) self.assertIsNotNone(enc_data) @@ -228,28 +227,9 @@ def check_decrypt(self, i): ctx = xmlsec.EncryptionContext(manager) decrypted = ctx.decrypt(enc_data) self.assertIsNotNone(decrypted) - self.assertEqual(self.load_xml("enc%d-in.xml" % i), root) + self.assertEqual(self.load_xml("enc{}-in.xml".format(i)), root) def test_decrypt_bad_args(self): ctx = xmlsec.EncryptionContext() with self.assertRaises(TypeError): ctx.decrypt('') - - def check_no_segfault(self): - namespaces = {'soap': 'http://schemas.xmlsoap.org/soap/envelope/'} - - manager = xmlsec.KeysManager() - key = xmlsec.Key.from_file(self.path("rsacert.pem"), format=consts.KeyDataFormatCertPem) - manager.add_key(key) - template = self.load_xml('enc-bad-in.xml') - enc_data = xmlsec.template.encrypted_data_create( - template, xmlsec.Transform.AES128, type=xmlsec.EncryptionType.CONTENT, ns='xenc' - ) - xmlsec.template.encrypted_data_ensure_cipher_value(enc_data) - key_info = xmlsec.template.encrypted_data_ensure_key_info(enc_data, ns='dsig') - enc_key = xmlsec.template.add_encrypted_key(key_info, xmlsec.Transform.RSA_PKCS1) - xmlsec.template.encrypted_data_ensure_cipher_value(enc_key) - data = template.find('soap:Body', namespaces=namespaces) - enc_ctx = xmlsec.EncryptionContext(manager) - enc_ctx.key = xmlsec.Key.generate(xmlsec.KeyData.AES, 192, xmlsec.KeyDataType.SESSION) - self.assertRaises(Exception, enc_ctx.encrypt_xml(enc_data, data)) diff --git a/tests/test_main.py b/tests/test_main.py index 9fb71eaf..3db18582 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -112,7 +112,7 @@ def test_sign_data_not_first_callback(self): def match_cb(filename): nonlocal bad_match_calls bad_match_calls += 1 - False + return False for _ in range(2): self._register_mismatch_callbacks(match_cb) @@ -132,7 +132,7 @@ def test_failed_sign_because_default_callbacks(self): def mismatch_cb(filename): nonlocal mismatch_calls mismatch_calls += 1 - False + return False # NB: These first two sets of callbacks should never get called, # because the default callbacks always match beforehand: diff --git a/tests/test_pkcs11.py b/tests/test_pkcs11.py new file mode 100644 index 00000000..accd29ae --- /dev/null +++ b/tests/test_pkcs11.py @@ -0,0 +1,57 @@ +import xmlsec +from tests import base +from xmlsec import constants as consts + +KEY_URL = "pkcs11;pkcs11:token=test;object=test;pin-value=secret1" + + +def setUpModule(): + from tests import softhsm_setup + + softhsm_setup.setup() + + +def tearDownModule(): + from tests import softhsm_setup + + softhsm_setup.teardown() + + +class TestKeys(base.TestMemoryLeaks): + def test_del_key(self): + ctx = xmlsec.SignatureContext(manager=xmlsec.KeysManager()) + ctx.key = xmlsec.Key.from_engine(KEY_URL) + del ctx.key + self.assertIsNone(ctx.key) + + def test_set_key(self): + ctx = xmlsec.SignatureContext(manager=xmlsec.KeysManager()) + ctx.key = xmlsec.Key.from_engine(KEY_URL) + self.assertIsNotNone(ctx.key) + + def test_sign_bad_args(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_engine(KEY_URL) + with self.assertRaises(TypeError): + ctx.sign('') + + def test_sign_fail(self): + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_engine(KEY_URL) + with self.assertRaisesRegex(xmlsec.Error, 'failed to sign'): + ctx.sign(self.load_xml('sign1-in.xml')) + + def test_sign_case1(self): + """Should sign a pre-constructed template file using a key from a pkcs11 engine.""" + root = self.load_xml("sign1-in.xml") + sign = xmlsec.tree.find_node(root, consts.NodeSignature) + self.assertIsNotNone(sign) + + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_engine(KEY_URL) + self.assertIsNotNone(ctx.key) + ctx.key.name = 'rsakey.pem' + self.assertEqual("rsakey.pem", ctx.key.name) + + ctx.sign(sign) + self.assertEqual(self.load_xml("sign1-out.xml"), root) diff --git a/tests/test_xmlsec.py b/tests/test_xmlsec.py index 32fac69a..303d7f8f 100644 --- a/tests/test_xmlsec.py +++ b/tests/test_xmlsec.py @@ -5,9 +5,9 @@ class TestModule(base.TestMemoryLeaks): def test_reinitialize_module(self): """ - This doesn't explicitly test anything, but will - be invoked first in the suite, so if the subsequent - tests don't fail, we know that the ``init()``/``shutdown()`` + This test doesn't explicitly verify anything, but will be invoked first in the suite. + + So if the subsequent tests don't fail, we know that the ``init()``/``shutdown()`` function pair doesn't break anything. """ xmlsec.shutdown()