From ea5044dda26f3ef18d46f139c4d9eb97baec5ba8 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 17 Jul 2023 13:28:38 -0700 Subject: [PATCH 001/120] Create script to update repo skeleton #80 Signed-off-by: Jono Yang --- etc/scripts/update_skeleton.py | 104 +++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 etc/scripts/update_skeleton.py diff --git a/etc/scripts/update_skeleton.py b/etc/scripts/update_skeleton.py new file mode 100644 index 00000000..635898ba --- /dev/null +++ b/etc/scripts/update_skeleton.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from pathlib import Path +import os +import subprocess + +import click + + +NEXB_PUBLIC_REPO_NAMES=[ + "aboutcode-toolkit", + "ahocode", + "bitcode", + "clearcode-toolkit", + "commoncode", + "container-inspector", + "debian-inspector", + "deltacode", + "elf-inspector", + "extractcode", + "fetchcode", + "gemfileparser2", + "gh-issue-sandbox", + "go-inspector", + "heritedcode", + "license-expression", + "license_copyright_pipeline", + "nuget-inspector", + "pip-requirements-parser", + "plugincode", + "purldb", + "pygmars", + "python-inspector", + "sanexml", + "saneyaml", + "scancode-analyzer", + "scancode-toolkit-contrib", + "scancode-toolkit-reference-scans", + "thirdparty-toolkit", + "tracecode-toolkit", + "tracecode-toolkit-strace", + "turbo-spdx", + "typecode", + "univers", +] + + +@click.command() +@click.help_option("-h", "--help") +def update_skeleton_files(repo_names=NEXB_PUBLIC_REPO_NAMES): + """ + Update project files of nexB projects that use the skeleton + + This script will: + - Clone the repo + - Add the skeleton repo as a new origin + - Create a new branch named "update-skeleton-files" + - Merge in the new skeleton files into the "update-skeleton-files" branch + + The user will need to save merge commit messages that pop up when running + this script in addition to resolving the merge conflicts on repos that have + them. + """ + + # Create working directory + work_dir_path = Path("/tmp/update_skeleton/") + if not os.path.exists(work_dir_path): + os.makedirs(work_dir_path, exist_ok=True) + + for repo_name in repo_names: + # Move to work directory + os.chdir(work_dir_path) + + # Clone repo + repo_git = f"git@github.com:nexB/{repo_name}.git" + subprocess.run(["git", "clone", repo_git]) + + # Go into cloned repo + os.chdir(work_dir_path / repo_name) + + # Add skeleton as an origin + subprocess.run(["git", "remote", "add", "skeleton", "git@github.com:nexB/skeleton.git"]) + + # Fetch skeleton files + subprocess.run(["git", "fetch", "skeleton"]) + + # Create and checkout new branch + subprocess.run(["git", "checkout", "-b", "update-skeleton-files"]) + + # Merge skeleton files into the repo + subprocess.run(["git", "merge", "skeleton/main", "--allow-unrelated-histories"]) + + +if __name__ == "__main__": + update_skeleton_files() From 5ab9b3a15e0095a8186ca3870cb9f9eade83833d Mon Sep 17 00:00:00 2001 From: Omkar Phansopkar Date: Wed, 18 Oct 2023 15:42:56 +0530 Subject: [PATCH 002/120] Added docs server script, dark mode & copybutton for docs Signed-off-by: Omkar Phansopkar --- .github/workflows/docs-ci.yml | 3 --- docs/Makefile | 8 ++++++++ docs/make.bat | 12 ++++++++++++ docs/scripts/doc8_style_check.sh | 0 docs/source/conf.py | 4 ++++ setup.cfg | 3 +++ 6 files changed, 27 insertions(+), 3 deletions(-) mode change 100644 => 100755 docs/scripts/doc8_style_check.sh diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 511b7c28..ada779bf 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -20,9 +20,6 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Give permission to run scripts - run: chmod +x ./docs/scripts/doc8_style_check.sh - - name: Install Dependencies run: pip install -e .[docs] diff --git a/docs/Makefile b/docs/Makefile index d0c3cbf1..788b0396 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -5,6 +5,7 @@ # from the environment for the first two. SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build +SPHINXAUTOBUILD = sphinx-autobuild SOURCEDIR = source BUILDDIR = build @@ -14,6 +15,13 @@ help: .PHONY: help Makefile +# Run the development server using sphinx-autobuild +docs: + @echo + @echo "Starting up the docs server..." + @echo + $(SPHINXAUTOBUILD) --port 8000 --watch ${SOURCEDIR} $(SOURCEDIR) "$(BUILDDIR)/html" $(SPHINXOPTS) $(O) + # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: Makefile diff --git a/docs/make.bat b/docs/make.bat index 6247f7e2..4a3c1a48 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -7,11 +7,16 @@ REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) +if "%SPHINXAUTOBUILD%" == "" ( + set SPHINXAUTOBUILD=sphinx-autobuild +) set SOURCEDIR=source set BUILDDIR=build if "%1" == "" goto help +if "%1" == "docs" goto docs + %SPHINXBUILD% >NUL 2>NUL if errorlevel 9009 ( echo. @@ -28,6 +33,13 @@ if errorlevel 9009 ( %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% goto end +:docs +@echo +@echo Starting up the docs server... +@echo +%SPHINXAUTOBUILD% --port 8000 --watch %SOURCEDIR% %SOURCEDIR% %BUILDDIR%\html %SPHINXOPTS% %O% +goto end + :help %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% diff --git a/docs/scripts/doc8_style_check.sh b/docs/scripts/doc8_style_check.sh old mode 100644 new mode 100755 diff --git a/docs/source/conf.py b/docs/source/conf.py index 918d62c1..54e5e665 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -30,6 +30,10 @@ extensions = [ "sphinx.ext.intersphinx", "sphinx_reredirects", + 'sphinx_rtd_theme', + "sphinx_rtd_dark_mode", + "sphinx.ext.extlinks", + "sphinx_copybutton", ] diff --git a/setup.cfg b/setup.cfg index d6c7da7d..bd0e58a7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -62,4 +62,7 @@ docs = sphinx-rtd-theme>=1.0.0 sphinx-reredirects >= 0.1.2 doc8>=0.11.2 + sphinx-autobuild + sphinx-rtd-dark-mode>=1.3.0 + sphinx-copybutton From 0a9d983650bf042a5bd2c277711b637979e566f1 Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Mon, 20 Nov 2023 16:46:54 -0800 Subject: [PATCH 003/120] Update CSS to widen page and handle mobile #84 Reference: https://github.com/nexB/skeleton/issues/84 Signed-off-by: John M. Horan --- docs/source/_static/theme_overrides.css | 363 +----------------- .../_static/theme_overrides_SUPERSEDED.css | 353 +++++++++++++++++ docs/source/conf.py | 15 +- 3 files changed, 380 insertions(+), 351 deletions(-) create mode 100644 docs/source/_static/theme_overrides_SUPERSEDED.css diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css index 9662d63a..de5ae433 100644 --- a/docs/source/_static/theme_overrides.css +++ b/docs/source/_static/theme_overrides.css @@ -1,353 +1,26 @@ -body { - color: #000000; -} - -p { - margin-bottom: 10px; -} - -.wy-plain-list-disc, .rst-content .section ul, .rst-content .toctree-wrapper ul, article ul { - margin-bottom: 10px; -} - -.custom_header_01 { - color: #cc0000; - font-size: 22px; - font-weight: bold; - line-height: 50px; -} - -h1, h2, h3, h4, h5, h6 { - margin-bottom: 20px; - margin-top: 20px; -} - -h5 { - font-size: 18px; - color: #000000; - font-style: italic; - margin-bottom: 10px; -} - -h6 { - font-size: 15px; - color: #000000; - font-style: italic; - margin-bottom: 10px; -} - -/* custom admonitions */ -/* success */ -.custom-admonition-success .admonition-title { - color: #000000; - background: #ccffcc; - border-radius: 5px 5px 0px 0px; -} -div.custom-admonition-success.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* important */ -.custom-admonition-important .admonition-title { - color: #000000; - background: #ccffcc; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #000000; -} -div.custom-admonition-important.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* caution */ -.custom-admonition-caution .admonition-title { - color: #000000; - background: #ffff99; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #e8e8e8; -} -div.custom-admonition-caution.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* note */ -.custom-admonition-note .admonition-title { - color: #ffffff; - background: #006bb3; - border-radius: 5px 5px 0px 0px; -} -div.custom-admonition-note.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* todo */ -.custom-admonition-todo .admonition-title { - color: #000000; - background: #cce6ff; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #99ccff; -} -div.custom-admonition-todo.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #99ccff; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* examples */ -.custom-admonition-examples .admonition-title { - color: #000000; - background: #ffe6cc; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #d8d8d8; -} -div.custom-admonition-examples.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - +/* this is the container for the pages */ .wy-nav-content { max-width: 100%; - padding-right: 100px; - padding-left: 100px; - background-color: #f2f2f2; -} - -div.rst-content { - background-color: #ffffff; - border: solid 1px #e5e5e5; - padding: 20px 40px 20px 40px; -} - -.rst-content .guilabel { - border: 1px solid #ffff99; - background: #ffff99; - font-size: 100%; - font-weight: normal; - border-radius: 4px; - padding: 2px 0px; - margin: auto 2px; - vertical-align: middle; -} - -.rst-content kbd { - font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; - border: solid 1px #d8d8d8; - background-color: #f5f5f5; - padding: 0px 3px; - border-radius: 3px; -} - -.wy-nav-content-wrap a { - color: #0066cc; - text-decoration: none; -} -.wy-nav-content-wrap a:hover { - color: #0099cc; - text-decoration: underline; -} - -.wy-nav-top a { - color: #ffffff; -} - -/* Based on numerous similar approaches e.g., https://github.com/readthedocs/sphinx_rtd_theme/issues/117 and https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html -- but remove form-factor limits to enable table wrap on full-size and smallest-size form factors */ -.wy-table-responsive table td { - white-space: normal !important; -} - -.rst-content table.docutils td, -.rst-content table.docutils th { - padding: 5px 10px 5px 10px; -} -.rst-content table.docutils td p, -.rst-content table.docutils th p { - font-size: 14px; - margin-bottom: 0px; -} -.rst-content table.docutils td p cite, -.rst-content table.docutils th p cite { - font-size: 14px; - background-color: transparent; -} - -.colwidths-given th { - border: solid 1px #d8d8d8 !important; -} -.colwidths-given td { - border: solid 1px #d8d8d8 !important; -} - -/*handles single-tick inline code*/ -.wy-body-for-nav cite { - color: #000000; - background-color: transparent; - font-style: normal; - font-family: "Courier New"; - font-size: 13px; - padding: 3px 3px 3px 3px; -} - -.rst-content pre.literal-block, .rst-content div[class^="highlight"] pre, .rst-content .linenodiv pre { - font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; - font-size: 13px; - overflow: visible; - white-space: pre-wrap; - color: #000000; -} - -.rst-content pre.literal-block, .rst-content div[class^='highlight'] { - background-color: #f8f8f8; - border: solid 1px #e8e8e8; -} - -/* This enables inline code to wrap. */ -code, .rst-content tt, .rst-content code { - white-space: pre-wrap; - padding: 2px 3px 1px; - border-radius: 3px; - font-size: 13px; - background-color: #ffffff; -} - -/* use this added class for code blocks attached to bulleted list items */ -.highlight-top-margin { - margin-top: 20px !important; -} - -/* change color of inline code block */ -span.pre { - color: #e01e5a; -} - -.wy-body-for-nav blockquote { - margin: 1em 0; - padding-left: 1em; - border-left: 4px solid #ddd; - color: #000000; -} - -/* Fix the unwanted top and bottom padding inside a nested bulleted/numbered list */ -.rst-content .section ol p, .rst-content .section ul p { - margin-bottom: 0px; -} - -/* add spacing between bullets for legibility */ -.rst-content .section ol li, .rst-content .section ul li { - margin-bottom: 5px; -} - -.rst-content .section ol li:first-child, .rst-content .section ul li:first-child { - margin-top: 5px; -} - -/* but exclude the toctree bullets */ -.rst-content .toctree-wrapper ul li, .rst-content .toctree-wrapper ul li:first-child { + padding: 0px 40px 0px 0px; margin-top: 0px; - margin-bottom: 0px; } -/* remove extra space at bottom of multine list-table cell */ -.rst-content .line-block { - margin-left: 0px; - margin-bottom: 0px; - line-height: 24px; +.wy-nav-content-wrap { + border-right: solid 1px; } -/* fix extra vertical spacing in page toctree */ -.rst-content .toctree-wrapper ul li ul, article ul li ul { - margin-top: 0; - margin-bottom: 0; -} - -/* this is used by the genindex added via layout.html (see source/_templates/) to sidebar toc */ -.reference.internal.toc-index { - color: #d9d9d9; -} - -.reference.internal.toc-index.current { - background-color: #ffffff; - color: #000000; - font-weight: bold; -} - -.toc-index-div { - border-top: solid 1px #000000; - margin-top: 10px; - padding-top: 5px; -} - -.indextable ul li { - font-size: 14px; - margin-bottom: 5px; -} - -/* The next 2 fix the poor vertical spacing in genindex.html (the alphabetized index) */ -.indextable.genindextable { - margin-bottom: 20px; -} - -div.genindex-jumpbox { - margin-bottom: 10px; -} - -/* rst image classes */ - -.clear-both { - clear: both; - } - -.float-left { - float: left; - margin-right: 20px; -} - -img { - border: solid 1px #e8e8e8; -} - -/* These are custom and need to be defined in conf.py to access in all pages, e.g., '.. role:: red' */ -.img-title { - color: #000000; - /* neither padding nor margin works for vertical spacing bc it's a span -- line-height does, sort of */ - line-height: 3.0; - font-style: italic; - font-weight: 600; -} - -.img-title-para { - color: #000000; - margin-top: 20px; - margin-bottom: 0px; - font-style: italic; - font-weight: 500; -} - -.red { - color: red; +div.rst-content { + max-width: 1300px; + border: 0; + padding: 0px 80px 10px 80px; + margin-left: 50px; +} + +@media (max-width: 768px) { + div.rst-content { + max-width: 1300px; + border: 0; + padding: 0px 10px 10px 10px; + margin-left: 0px; + } } diff --git a/docs/source/_static/theme_overrides_SUPERSEDED.css b/docs/source/_static/theme_overrides_SUPERSEDED.css new file mode 100644 index 00000000..9662d63a --- /dev/null +++ b/docs/source/_static/theme_overrides_SUPERSEDED.css @@ -0,0 +1,353 @@ +body { + color: #000000; +} + +p { + margin-bottom: 10px; +} + +.wy-plain-list-disc, .rst-content .section ul, .rst-content .toctree-wrapper ul, article ul { + margin-bottom: 10px; +} + +.custom_header_01 { + color: #cc0000; + font-size: 22px; + font-weight: bold; + line-height: 50px; +} + +h1, h2, h3, h4, h5, h6 { + margin-bottom: 20px; + margin-top: 20px; +} + +h5 { + font-size: 18px; + color: #000000; + font-style: italic; + margin-bottom: 10px; +} + +h6 { + font-size: 15px; + color: #000000; + font-style: italic; + margin-bottom: 10px; +} + +/* custom admonitions */ +/* success */ +.custom-admonition-success .admonition-title { + color: #000000; + background: #ccffcc; + border-radius: 5px 5px 0px 0px; +} +div.custom-admonition-success.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* important */ +.custom-admonition-important .admonition-title { + color: #000000; + background: #ccffcc; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #000000; +} +div.custom-admonition-important.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* caution */ +.custom-admonition-caution .admonition-title { + color: #000000; + background: #ffff99; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #e8e8e8; +} +div.custom-admonition-caution.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* note */ +.custom-admonition-note .admonition-title { + color: #ffffff; + background: #006bb3; + border-radius: 5px 5px 0px 0px; +} +div.custom-admonition-note.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* todo */ +.custom-admonition-todo .admonition-title { + color: #000000; + background: #cce6ff; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #99ccff; +} +div.custom-admonition-todo.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #99ccff; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +/* examples */ +.custom-admonition-examples .admonition-title { + color: #000000; + background: #ffe6cc; + border-radius: 5px 5px 0px 0px; + border-bottom: solid 1px #d8d8d8; +} +div.custom-admonition-examples.admonition { + color: #000000; + background: #ffffff; + border: solid 1px #cccccc; + border-radius: 5px; + box-shadow: 1px 1px 5px 3px #d8d8d8; + margin: 20px 0px 30px 0px; +} + +.wy-nav-content { + max-width: 100%; + padding-right: 100px; + padding-left: 100px; + background-color: #f2f2f2; +} + +div.rst-content { + background-color: #ffffff; + border: solid 1px #e5e5e5; + padding: 20px 40px 20px 40px; +} + +.rst-content .guilabel { + border: 1px solid #ffff99; + background: #ffff99; + font-size: 100%; + font-weight: normal; + border-radius: 4px; + padding: 2px 0px; + margin: auto 2px; + vertical-align: middle; +} + +.rst-content kbd { + font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; + border: solid 1px #d8d8d8; + background-color: #f5f5f5; + padding: 0px 3px; + border-radius: 3px; +} + +.wy-nav-content-wrap a { + color: #0066cc; + text-decoration: none; +} +.wy-nav-content-wrap a:hover { + color: #0099cc; + text-decoration: underline; +} + +.wy-nav-top a { + color: #ffffff; +} + +/* Based on numerous similar approaches e.g., https://github.com/readthedocs/sphinx_rtd_theme/issues/117 and https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html -- but remove form-factor limits to enable table wrap on full-size and smallest-size form factors */ +.wy-table-responsive table td { + white-space: normal !important; +} + +.rst-content table.docutils td, +.rst-content table.docutils th { + padding: 5px 10px 5px 10px; +} +.rst-content table.docutils td p, +.rst-content table.docutils th p { + font-size: 14px; + margin-bottom: 0px; +} +.rst-content table.docutils td p cite, +.rst-content table.docutils th p cite { + font-size: 14px; + background-color: transparent; +} + +.colwidths-given th { + border: solid 1px #d8d8d8 !important; +} +.colwidths-given td { + border: solid 1px #d8d8d8 !important; +} + +/*handles single-tick inline code*/ +.wy-body-for-nav cite { + color: #000000; + background-color: transparent; + font-style: normal; + font-family: "Courier New"; + font-size: 13px; + padding: 3px 3px 3px 3px; +} + +.rst-content pre.literal-block, .rst-content div[class^="highlight"] pre, .rst-content .linenodiv pre { + font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; + font-size: 13px; + overflow: visible; + white-space: pre-wrap; + color: #000000; +} + +.rst-content pre.literal-block, .rst-content div[class^='highlight'] { + background-color: #f8f8f8; + border: solid 1px #e8e8e8; +} + +/* This enables inline code to wrap. */ +code, .rst-content tt, .rst-content code { + white-space: pre-wrap; + padding: 2px 3px 1px; + border-radius: 3px; + font-size: 13px; + background-color: #ffffff; +} + +/* use this added class for code blocks attached to bulleted list items */ +.highlight-top-margin { + margin-top: 20px !important; +} + +/* change color of inline code block */ +span.pre { + color: #e01e5a; +} + +.wy-body-for-nav blockquote { + margin: 1em 0; + padding-left: 1em; + border-left: 4px solid #ddd; + color: #000000; +} + +/* Fix the unwanted top and bottom padding inside a nested bulleted/numbered list */ +.rst-content .section ol p, .rst-content .section ul p { + margin-bottom: 0px; +} + +/* add spacing between bullets for legibility */ +.rst-content .section ol li, .rst-content .section ul li { + margin-bottom: 5px; +} + +.rst-content .section ol li:first-child, .rst-content .section ul li:first-child { + margin-top: 5px; +} + +/* but exclude the toctree bullets */ +.rst-content .toctree-wrapper ul li, .rst-content .toctree-wrapper ul li:first-child { + margin-top: 0px; + margin-bottom: 0px; +} + +/* remove extra space at bottom of multine list-table cell */ +.rst-content .line-block { + margin-left: 0px; + margin-bottom: 0px; + line-height: 24px; +} + +/* fix extra vertical spacing in page toctree */ +.rst-content .toctree-wrapper ul li ul, article ul li ul { + margin-top: 0; + margin-bottom: 0; +} + +/* this is used by the genindex added via layout.html (see source/_templates/) to sidebar toc */ +.reference.internal.toc-index { + color: #d9d9d9; +} + +.reference.internal.toc-index.current { + background-color: #ffffff; + color: #000000; + font-weight: bold; +} + +.toc-index-div { + border-top: solid 1px #000000; + margin-top: 10px; + padding-top: 5px; +} + +.indextable ul li { + font-size: 14px; + margin-bottom: 5px; +} + +/* The next 2 fix the poor vertical spacing in genindex.html (the alphabetized index) */ +.indextable.genindextable { + margin-bottom: 20px; +} + +div.genindex-jumpbox { + margin-bottom: 10px; +} + +/* rst image classes */ + +.clear-both { + clear: both; + } + +.float-left { + float: left; + margin-right: 20px; +} + +img { + border: solid 1px #e8e8e8; +} + +/* These are custom and need to be defined in conf.py to access in all pages, e.g., '.. role:: red' */ +.img-title { + color: #000000; + /* neither padding nor margin works for vertical spacing bc it's a span -- line-height does, sort of */ + line-height: 3.0; + font-style: italic; + font-weight: 600; +} + +.img-title-para { + color: #000000; + margin-top: 20px; + margin-bottom: 0px; + font-style: italic; + font-weight: 500; +} + +.red { + color: red; +} diff --git a/docs/source/conf.py b/docs/source/conf.py index 54e5e665..7771ff09 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -30,7 +30,7 @@ extensions = [ "sphinx.ext.intersphinx", "sphinx_reredirects", - 'sphinx_rtd_theme', + "sphinx_rtd_theme", "sphinx_rtd_dark_mode", "sphinx.ext.extlinks", "sphinx_copybutton", @@ -47,7 +47,10 @@ intersphinx_mapping = { "aboutcode": ("https://aboutcode.readthedocs.io/en/latest/", None), - "scancode-workbench": ("https://scancode-workbench.readthedocs.io/en/develop/", None), + "scancode-workbench": ( + "https://scancode-workbench.readthedocs.io/en/develop/", + None, + ), } @@ -82,7 +85,9 @@ "conf_py_path": "/docs/source/", # path in the checkout to the docs root } -html_css_files = ["_static/theme_overrides.css"] +html_css_files = [ + "theme_overrides.css", +] # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. @@ -108,6 +113,4 @@ # -- Options for LaTeX output ------------------------------------------------- -latex_elements = { - 'classoptions': ',openany,oneside' -} \ No newline at end of file +latex_elements = {"classoptions": ",openany,oneside"} From 4e36fc601eaa17bde0d2a4bebfed70d7bde28e7c Mon Sep 17 00:00:00 2001 From: "John M. Horan" Date: Tue, 16 Jan 2024 12:22:54 -0800 Subject: [PATCH 004/120] Delete theme_overrides_SUPERSEDED.css as no longer needed #84 Reference: https://github.com/nexB/skeleton/issues/84 Signed-off-by: John M. Horan --- .../_static/theme_overrides_SUPERSEDED.css | 353 ------------------ 1 file changed, 353 deletions(-) delete mode 100644 docs/source/_static/theme_overrides_SUPERSEDED.css diff --git a/docs/source/_static/theme_overrides_SUPERSEDED.css b/docs/source/_static/theme_overrides_SUPERSEDED.css deleted file mode 100644 index 9662d63a..00000000 --- a/docs/source/_static/theme_overrides_SUPERSEDED.css +++ /dev/null @@ -1,353 +0,0 @@ -body { - color: #000000; -} - -p { - margin-bottom: 10px; -} - -.wy-plain-list-disc, .rst-content .section ul, .rst-content .toctree-wrapper ul, article ul { - margin-bottom: 10px; -} - -.custom_header_01 { - color: #cc0000; - font-size: 22px; - font-weight: bold; - line-height: 50px; -} - -h1, h2, h3, h4, h5, h6 { - margin-bottom: 20px; - margin-top: 20px; -} - -h5 { - font-size: 18px; - color: #000000; - font-style: italic; - margin-bottom: 10px; -} - -h6 { - font-size: 15px; - color: #000000; - font-style: italic; - margin-bottom: 10px; -} - -/* custom admonitions */ -/* success */ -.custom-admonition-success .admonition-title { - color: #000000; - background: #ccffcc; - border-radius: 5px 5px 0px 0px; -} -div.custom-admonition-success.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* important */ -.custom-admonition-important .admonition-title { - color: #000000; - background: #ccffcc; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #000000; -} -div.custom-admonition-important.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* caution */ -.custom-admonition-caution .admonition-title { - color: #000000; - background: #ffff99; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #e8e8e8; -} -div.custom-admonition-caution.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* note */ -.custom-admonition-note .admonition-title { - color: #ffffff; - background: #006bb3; - border-radius: 5px 5px 0px 0px; -} -div.custom-admonition-note.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* todo */ -.custom-admonition-todo .admonition-title { - color: #000000; - background: #cce6ff; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #99ccff; -} -div.custom-admonition-todo.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #99ccff; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -/* examples */ -.custom-admonition-examples .admonition-title { - color: #000000; - background: #ffe6cc; - border-radius: 5px 5px 0px 0px; - border-bottom: solid 1px #d8d8d8; -} -div.custom-admonition-examples.admonition { - color: #000000; - background: #ffffff; - border: solid 1px #cccccc; - border-radius: 5px; - box-shadow: 1px 1px 5px 3px #d8d8d8; - margin: 20px 0px 30px 0px; -} - -.wy-nav-content { - max-width: 100%; - padding-right: 100px; - padding-left: 100px; - background-color: #f2f2f2; -} - -div.rst-content { - background-color: #ffffff; - border: solid 1px #e5e5e5; - padding: 20px 40px 20px 40px; -} - -.rst-content .guilabel { - border: 1px solid #ffff99; - background: #ffff99; - font-size: 100%; - font-weight: normal; - border-radius: 4px; - padding: 2px 0px; - margin: auto 2px; - vertical-align: middle; -} - -.rst-content kbd { - font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; - border: solid 1px #d8d8d8; - background-color: #f5f5f5; - padding: 0px 3px; - border-radius: 3px; -} - -.wy-nav-content-wrap a { - color: #0066cc; - text-decoration: none; -} -.wy-nav-content-wrap a:hover { - color: #0099cc; - text-decoration: underline; -} - -.wy-nav-top a { - color: #ffffff; -} - -/* Based on numerous similar approaches e.g., https://github.com/readthedocs/sphinx_rtd_theme/issues/117 and https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html -- but remove form-factor limits to enable table wrap on full-size and smallest-size form factors */ -.wy-table-responsive table td { - white-space: normal !important; -} - -.rst-content table.docutils td, -.rst-content table.docutils th { - padding: 5px 10px 5px 10px; -} -.rst-content table.docutils td p, -.rst-content table.docutils th p { - font-size: 14px; - margin-bottom: 0px; -} -.rst-content table.docutils td p cite, -.rst-content table.docutils th p cite { - font-size: 14px; - background-color: transparent; -} - -.colwidths-given th { - border: solid 1px #d8d8d8 !important; -} -.colwidths-given td { - border: solid 1px #d8d8d8 !important; -} - -/*handles single-tick inline code*/ -.wy-body-for-nav cite { - color: #000000; - background-color: transparent; - font-style: normal; - font-family: "Courier New"; - font-size: 13px; - padding: 3px 3px 3px 3px; -} - -.rst-content pre.literal-block, .rst-content div[class^="highlight"] pre, .rst-content .linenodiv pre { - font-family: SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",Courier,monospace; - font-size: 13px; - overflow: visible; - white-space: pre-wrap; - color: #000000; -} - -.rst-content pre.literal-block, .rst-content div[class^='highlight'] { - background-color: #f8f8f8; - border: solid 1px #e8e8e8; -} - -/* This enables inline code to wrap. */ -code, .rst-content tt, .rst-content code { - white-space: pre-wrap; - padding: 2px 3px 1px; - border-radius: 3px; - font-size: 13px; - background-color: #ffffff; -} - -/* use this added class for code blocks attached to bulleted list items */ -.highlight-top-margin { - margin-top: 20px !important; -} - -/* change color of inline code block */ -span.pre { - color: #e01e5a; -} - -.wy-body-for-nav blockquote { - margin: 1em 0; - padding-left: 1em; - border-left: 4px solid #ddd; - color: #000000; -} - -/* Fix the unwanted top and bottom padding inside a nested bulleted/numbered list */ -.rst-content .section ol p, .rst-content .section ul p { - margin-bottom: 0px; -} - -/* add spacing between bullets for legibility */ -.rst-content .section ol li, .rst-content .section ul li { - margin-bottom: 5px; -} - -.rst-content .section ol li:first-child, .rst-content .section ul li:first-child { - margin-top: 5px; -} - -/* but exclude the toctree bullets */ -.rst-content .toctree-wrapper ul li, .rst-content .toctree-wrapper ul li:first-child { - margin-top: 0px; - margin-bottom: 0px; -} - -/* remove extra space at bottom of multine list-table cell */ -.rst-content .line-block { - margin-left: 0px; - margin-bottom: 0px; - line-height: 24px; -} - -/* fix extra vertical spacing in page toctree */ -.rst-content .toctree-wrapper ul li ul, article ul li ul { - margin-top: 0; - margin-bottom: 0; -} - -/* this is used by the genindex added via layout.html (see source/_templates/) to sidebar toc */ -.reference.internal.toc-index { - color: #d9d9d9; -} - -.reference.internal.toc-index.current { - background-color: #ffffff; - color: #000000; - font-weight: bold; -} - -.toc-index-div { - border-top: solid 1px #000000; - margin-top: 10px; - padding-top: 5px; -} - -.indextable ul li { - font-size: 14px; - margin-bottom: 5px; -} - -/* The next 2 fix the poor vertical spacing in genindex.html (the alphabetized index) */ -.indextable.genindextable { - margin-bottom: 20px; -} - -div.genindex-jumpbox { - margin-bottom: 10px; -} - -/* rst image classes */ - -.clear-both { - clear: both; - } - -.float-left { - float: left; - margin-right: 20px; -} - -img { - border: solid 1px #e8e8e8; -} - -/* These are custom and need to be defined in conf.py to access in all pages, e.g., '.. role:: red' */ -.img-title { - color: #000000; - /* neither padding nor margin works for vertical spacing bc it's a span -- line-height does, sort of */ - line-height: 3.0; - font-style: italic; - font-weight: 600; -} - -.img-title-para { - color: #000000; - margin-top: 20px; - margin-bottom: 0px; - font-style: italic; - font-weight: 500; -} - -.red { - color: red; -} From 7d74b8a3c98761293cd133d543e4d58a525dc7bf Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 18 Jan 2024 17:11:14 +0530 Subject: [PATCH 005/120] Fix top padding for rst content Signed-off-by: Ayan Sinha Mahapatra --- docs/source/_static/theme_overrides.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css index de5ae433..5863ccf5 100644 --- a/docs/source/_static/theme_overrides.css +++ b/docs/source/_static/theme_overrides.css @@ -12,7 +12,7 @@ div.rst-content { max-width: 1300px; border: 0; - padding: 0px 80px 10px 80px; + padding: 10px 80px 10px 80px; margin-left: 50px; } From 008d521aec51e5983f6d6a2adc4efa7fd92159cf Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 19 Feb 2024 15:21:45 +0530 Subject: [PATCH 006/120] Update CI runners and python version Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/docs-ci.yml | 2 +- .github/workflows/pypi-release.yml | 8 ++++---- azure-pipelines.yml | 22 +++++++++++++++------- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index ada779bf..8c2abfe9 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 strategy: max-parallel: 4 diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 95857301..d2206c87 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -21,10 +21,10 @@ on: jobs: build-pypi-distribs: name: Build and publish library to PyPI - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: @@ -47,7 +47,7 @@ jobs: name: Create GH release needs: - build-pypi-distribs - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Download built archives @@ -67,7 +67,7 @@ jobs: name: Create PyPI release needs: - create-gh-release - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - name: Download built archives diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 764883de..373b78cd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -11,7 +11,7 @@ jobs: parameters: job_name: ubuntu20_cpython image_name: ubuntu-20.04 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -19,7 +19,7 @@ jobs: parameters: job_name: ubuntu22_cpython image_name: ubuntu-22.04 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -27,7 +27,7 @@ jobs: parameters: job_name: macos11_cpython image_name: macOS-11 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -35,7 +35,7 @@ jobs: parameters: job_name: macos12_cpython image_name: macOS-12 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +43,15 @@ jobs: parameters: job_name: macos13_cpython image_name: macOS-13 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos14_cpython + image_name: macOS-14 + python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -51,7 +59,7 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -59,6 +67,6 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From 5d645a26e7acc952eeb0da954f7b303d8d42d634 Mon Sep 17 00:00:00 2001 From: stefan6419846 <96178532+stefan6419846@users.noreply.github.com> Date: Sat, 13 Jan 2024 10:07:24 +0100 Subject: [PATCH 007/120] Replace the deprecated datetime.datetime.utcfromtimestamp Signed-off-by: stefan6419846 <96178532+stefan6419846@users.noreply.github.com> --- src/commoncode/filetype.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/commoncode/filetype.py b/src/commoncode/filetype.py index 3f2201b8..7f7501a5 100644 --- a/src/commoncode/filetype.py +++ b/src/commoncode/filetype.py @@ -7,7 +7,7 @@ # import os -from datetime import datetime +from datetime import datetime, timezone from commoncode.system import on_posix from commoncode.functional import memoize @@ -164,7 +164,10 @@ def get_last_modified_date(location): yyyymmdd = '' if is_file(location): utc_date = datetime.isoformat( - datetime.utcfromtimestamp(os.path.getmtime(location)) + datetime.fromtimestamp( + os.path.getmtime(location), + tz=timezone.utc, + ) ) yyyymmdd = utc_date[:10] return yyyymmdd From 124da3dcef0d95a6f6aa76ed849f47ada25b83e2 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 1 Jul 2024 15:11:21 +0530 Subject: [PATCH 008/120] Replace deprecated macos CI runners Replace macos-11 runners with macos-14 runners. Reference: https://github.com/actions/runner-images?tab=readme-ov-file#available-images Reference: https://github.com/nexB/skeleton/issues/89 Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 373b78cd..c2a3b522 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -25,24 +25,24 @@ jobs: - template: etc/ci/azure-posix.yml parameters: - job_name: macos11_cpython - image_name: macOS-11 + job_name: macos12_cpython + image_name: macOS-12 python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-posix.yml parameters: - job_name: macos12_cpython - image_name: macOS-12 + job_name: macos13_cpython + image_name: macOS-13 python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-posix.yml parameters: - job_name: macos13_cpython - image_name: macOS-13 + job_name: macos14_cpython_arm64 + image_name: macOS-14 python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -50,8 +50,8 @@ jobs: - template: etc/ci/azure-posix.yml parameters: job_name: macos14_cpython - image_name: macOS-14 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + image_name: macOS-14-large + python_versions: ['3.8', '3.8', '3.9', '3.10', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs From be4e14d414cf4f7112b529dc71f7abccc9dcf24a Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Mon, 1 Jul 2024 16:00:40 +0530 Subject: [PATCH 009/120] Update minimum required python version to 3.8 Signed-off-by: Ayan Sinha Mahapatra --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index bd0e58a7..a8e20c5d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -38,7 +38,7 @@ zip_safe = false setup_requires = setuptools_scm[toml] >= 4 -python_requires = >=3.7 +python_requires = >=3.8 install_requires = From 4f87b3c9272dcf209b9c4b997e98b58e0edaf570 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=BCtz?= Date: Wed, 24 Jul 2024 14:14:30 -0700 Subject: [PATCH 010/120] don't fail if there is no /etc/os-release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Robert Schütz --- src/commoncode/system.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/commoncode/system.py b/src/commoncode/system.py index 07938f9b..58359992 100644 --- a/src/commoncode/system.py +++ b/src/commoncode/system.py @@ -94,7 +94,10 @@ def is_on_ubuntu_22(os_release_location='/etc/os-release'): os_release_info = parse_os_release(os_release_location) return os_release_info['ID'] == 'ubuntu' and '22' in os_release_info['VERSION_ID'] -on_ubuntu_22 = is_on_ubuntu_22() +try: + on_ubuntu_22 = is_on_ubuntu_22() +except FileNotFoundError: + on_ubuntu_22 = False del is_on_ubuntu_22 From dd4d26a5212fd4a6315dff9ae9866bafc6e694e8 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Wed, 14 Aug 2024 12:25:11 +0800 Subject: [PATCH 011/120] Remove the deprecated macos_11 Signed-off-by: Chin Yeung Li --- azure-pipelines.yml | 213 +++++++++++++++++++++----------------------- 1 file changed, 102 insertions(+), 111 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 02c7530c..a5aab8e6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,4 +1,3 @@ - ################################################################################ # We use Azure to run the full tests suites on multiple Python 3.x # on multiple Windows, macOS and Linux versions all on 64 bits @@ -6,113 +5,105 @@ ################################################################################ jobs: - -################################################################################ -# These jobs are using VMs and Azure-provided Pythons 3.8 -################################################################################ - - - template: etc/ci/azure-posix.yml - parameters: - job_name: ubuntu20_cpython - image_name: ubuntu-20.04 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - - template: etc/ci/azure-posix.yml - parameters: - job_name: ubuntu22_cpython - image_name: ubuntu-22.04 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos11_cpython - image_name: macOS-11 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos12_cpython - image_name: macOS-12 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos13_cpython - image_name: macOS-13 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - - template: etc/ci/azure-win.yml - parameters: - job_name: win2019_cpython - image_name: windows-2019 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] - test_suites: - all: venv\Scripts\pytest -n 2 -vvs - - - template: etc/ci/azure-win.yml - parameters: - job_name: win2022_cpython - image_name: windows-2022 - python_versions: ['3.7', '3.8', '3.9', '3.10', '3.11'] - test_suites: - all: venv\Scripts\pytest -n 2 -vvs - - -################################################################################ -# Test using many version of Click to work around regressions in API -################################################################################ - - - - template: etc/ci/azure-posix.yml - parameters: - job_name: ubuntu20_test_all_supported_click_versions - image_name: ubuntu-20.04 - python_versions: ['3.7', '3.8', '3.9', '3.10'] - test_suites: - click_versions: | - for clk_ver in 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7; - do - pip install click==$clk_ver; - venv/bin/pytest -vvs tests/test_cliutils_progressbar.py; - done - - -################################################################################ -# Tests using a plain pip install to get the latest of all wheels -################################################################################ - - - - template: etc/ci/azure-posix.yml - parameters: - job_name: ubuntu20_cpython_latest_from_pip - image_name: ubuntu-20.04 - python_versions: ['3.7', '3.8', '3.9', '3.10'] - test_suites: - all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e . && venv/bin/pytest -n 2 -vvs - - - - template: etc/ci/azure-win.yml - parameters: - job_name: win2019_cpython_latest_from_pip - image_name: windows-2019 - python_versions: ['3.7', '3.8', '3.9', '3.10'] - test_suites: - all: venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e . && venv\Scripts\pytest -n 2 -vvs - - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos11_cpython_latest_from_pip - image_name: macos-11 - python_versions: ['3.7', '3.8', '3.9', '3.10'] - test_suites: - all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e . && venv/bin/pytest -n 2 -vvs + ################################################################################ + # These jobs are using VMs and Azure-provided Pythons 3.8 + ################################################################################ + + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu20_cpython + image_name: ubuntu-20.04 + python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu22_cpython + image_name: ubuntu-22.04 + python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos12_cpython + image_name: macOS-12 + python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos13_cpython + image_name: macOS-13 + python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2019_cpython + image_name: windows-2019 + python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + test_suites: + all: venv\Scripts\pytest -n 2 -vvs + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2022_cpython + image_name: windows-2022 + python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + test_suites: + all: venv\Scripts\pytest -n 2 -vvs + + ################################################################################ + # Test using many version of Click to work around regressions in API + ################################################################################ + + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu20_test_all_supported_click_versions + image_name: ubuntu-20.04 + python_versions: ["3.7", "3.8", "3.9", "3.10"] + test_suites: + click_versions: | + for clk_ver in 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7; + do + pip install click==$clk_ver; + venv/bin/pytest -vvs tests/test_cliutils_progressbar.py; + done + + ################################################################################ + # Tests using a plain pip install to get the latest of all wheels + ################################################################################ + + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu20_cpython_latest_from_pip + image_name: ubuntu-20.04 + python_versions: ["3.7", "3.8", "3.9", "3.10"] + test_suites: + all: + venv/bin/pip install --upgrade-strategy eager --force-reinstall + --upgrade -e . && venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2019_cpython_latest_from_pip + image_name: windows-2019 + python_versions: ["3.7", "3.8", "3.9", "3.10"] + test_suites: + all: + venv\Scripts\pip install --upgrade-strategy eager --force-reinstall + --upgrade -e . && venv\Scripts\pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos11_cpython_latest_from_pip + image_name: macos-11 + python_versions: ["3.7", "3.8", "3.9", "3.10"] + test_suites: + all: + venv/bin/pip install --upgrade-strategy eager --force-reinstall + --upgrade -e . && venv/bin/pytest -n 2 -vvs From a1bffad80661b4c5b155aa79001ea38bd2dbbc20 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Wed, 14 Aug 2024 12:29:27 +0800 Subject: [PATCH 012/120] #67 - Update link references of ownership from nexB to aboutcode-org Signed-off-by: Chin Yeung Li --- NOTICE | 2 +- README.rst | 4 +- commoncode.ABOUT | 4 +- configure | 2 +- configure.bat | 2 +- setup.cfg | 2 +- src/commoncode/__init__.py | 2 +- src/commoncode/archive.py | 2 +- src/commoncode/cliutils.py | 30 ++-- src/commoncode/codec.py | 7 +- src/commoncode/command.py | 14 +- src/commoncode/compat.py | 2 +- src/commoncode/datautils.py | 2 +- src/commoncode/date.py | 4 +- src/commoncode/distro.py | 2 +- src/commoncode/fetch.py | 5 +- src/commoncode/fileset.py | 14 +- src/commoncode/filetype.py | 2 +- src/commoncode/fileutils.py | 4 +- src/commoncode/functional.py | 2 +- src/commoncode/hash.py | 2 +- src/commoncode/ignore.py | 12 +- src/commoncode/paths.py | 5 +- src/commoncode/resource.py | 91 ++++++++---- src/commoncode/system.py | 3 +- src/commoncode/testcase.py | 21 ++- src/commoncode/text.py | 2 +- src/commoncode/timeutils.py | 2 +- src/commoncode/urn.py | 2 +- src/commoncode/version.py | 28 ++-- tests/test_cliutils.py | 27 ++-- tests/test_cliutils_progressbar.py | 3 +- tests/test_codec.py | 4 +- tests/test_command.py | 5 +- tests/test_date.py | 2 +- tests/test_fileset.py | 8 +- tests/test_filetype.py | 6 +- tests/test_fileutils.py | 67 ++++++--- tests/test_functional.py | 2 +- tests/test_hash.py | 11 +- tests/test_ignore.py | 41 ++++-- tests/test_paths.py | 50 ++++--- tests/test_resource.py | 227 +++++++++++++++++++---------- tests/test_saneyaml.py | 2 +- tests/test_text.py | 8 +- tests/test_timeutils.py | 29 ++-- tests/test_urn.py | 8 +- tests/test_version.py | 4 +- 48 files changed, 485 insertions(+), 295 deletions(-) diff --git a/NOTICE b/NOTICE index 2643d9b1..b5a2017b 100644 --- a/NOTICE +++ b/NOTICE @@ -2,6 +2,6 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/README.rst b/README.rst index c2106da3..654b53e8 100644 --- a/README.rst +++ b/README.rst @@ -3,12 +3,12 @@ CommonCode - license: Apache-2.0 - copyright: copyright (c) nexB. Inc. and others -- homepage_url: https://github.com/nexB/commoncode +- homepage_url: https://github.com/aboutcode-org/commoncode - keywords: utilities, scancode-toolkit, commoncode Commoncode provides a set of common functions and utilities for handling various things like paths, dates, files and hashes. It started as library in scancode-toolkit. -Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +Visit https://aboutcode.org and https://github.com/aboutcode-org/ for support and download. To install this package use:: diff --git a/commoncode.ABOUT b/commoncode.ABOUT index 80578504..30f0e029 100644 --- a/commoncode.ABOUT +++ b/commoncode.ABOUT @@ -4,10 +4,10 @@ description: Commoncode provides a set of common functions and utilities for handling various things like paths, dates, files and hashes. It started as library in scancode-toolkit. keywords: utilities, commoncode, scancode-toolkit -homepage_url: https://github.com/nexB/commoncode +homepage_url: https://github.com/aboutcode-org/commoncode holder: nexB. Inc. and others holder_contact: info@aboutcode.org -homepage_url: https://github.com/nexB/commoncode +homepage_url: https://github.com/aboutcode-org/commoncode license_expression: apache-2.0 name: commoncode package_url: pkg:pypi/commoncode diff --git a/configure b/configure index 926a894e..22d92885 100755 --- a/configure +++ b/configure @@ -3,7 +3,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/ for support or download. +# See https://github.com/aboutcode-org/ for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/configure.bat b/configure.bat index 5e95b311..5b9a9d68 100644 --- a/configure.bat +++ b/configure.bat @@ -4,7 +4,7 @@ @rem Copyright (c) nexB Inc. and others. All rights reserved. @rem SPDX-License-Identifier: Apache-2.0 @rem See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -@rem See https://github.com/nexB/ for support or download. +@rem See https://github.com/aboutcode-org/ for support or download. @rem See https://aboutcode.org for more information about nexB OSS projects. diff --git a/setup.cfg b/setup.cfg index 984a1f1f..d7361f83 100644 --- a/setup.cfg +++ b/setup.cfg @@ -15,7 +15,7 @@ license = Apache-2.0 description = Set of common utilities, originally split from ScanCode long_description = file:README.rst long_description_content_type = text/x-rst -url = https://github.com/nexB/commoncode +url = https://github.com/aboutcode-org/commoncode classifiers = Development Status :: 5 - Production/Stable Intended Audience :: Developers diff --git a/src/commoncode/__init__.py b/src/commoncode/__init__.py index b3c06862..29888e4d 100644 --- a/src/commoncode/__init__.py +++ b/src/commoncode/__init__.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/src/commoncode/archive.py b/src/commoncode/archive.py index 01b7e0ba..671c4517 100644 --- a/src/commoncode/archive.py +++ b/src/commoncode/archive.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/src/commoncode/cliutils.py b/src/commoncode/cliutils.py index 05357bfb..b58736fc 100644 --- a/src/commoncode/cliutils.py +++ b/src/commoncode/cliutils.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -154,18 +154,18 @@ def format_options(self, ctx, formatter): # overriden and copied from Click to work around Click woes for -# https://github.com/nexB/scancode-toolkit/issues/2583 +# https://github.com/aboutcode-org/scancode-toolkit/issues/2583 class DebuggedProgressBar(ProgressBar): # overriden and copied from Click to work around Click woes for - # https://github.com/nexB/scancode-toolkit/issues/2583 + # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 def make_step(self, n_steps): # always increment self.pos += n_steps or 1 super(DebuggedProgressBar, self).make_step(n_steps) # overriden and copied from Click to work around Click woes for - # https://github.com/nexB/scancode-toolkit/issues/2583 + # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 def generator(self): if self.is_hidden: yield from self.iter @@ -290,7 +290,7 @@ def progressmanager( # Click 8. See https://github.com/pallets/click/pull/1698 # Note that we use this argument on Click 8 in order to fix a regression # that this same PR introduced by Click and tracked originally at - # https://github.com/nexB/scancode-toolkit/issues/2583 + # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 # Here we create a dummy progress_class and then for the attribute presence. pb = progress_class([]) if hasattr(pb, 'update_min_steps'): @@ -328,7 +328,7 @@ def fixed_width_file_name(path, max_length=25): len_ext = len(ext) remaining_length = max_length - len_ext - dots - if remaining_length < 5 or remaining_length < (len_ext + dots): + if remaining_length < 5 or remaining_length < (len_ext + dots): return '' prefix_and_suffix_length = abs(remaining_length // 2) @@ -473,8 +473,10 @@ def validate_dependencies(self, ctx, value): Validate `value` against declared `required_options` or `conflicting_options` dependencies. """ - _validate_option_dependencies(ctx, self, value, self.required_options, required=True) - _validate_option_dependencies(ctx, self, value, self.conflicting_options, required=False) + _validate_option_dependencies( + ctx, self, value, self.required_options, required=True) + _validate_option_dependencies( + ctx, self, value, self.conflicting_options, required=False) def get_help_record(self, ctx): if not self.hidden: @@ -497,11 +499,13 @@ def validate_option_dependencies(ctx): continue if not isinstance(param, PluggableCommandLineOption): if TRACE: - logger_debug(' validate_option_dependencies: skip param:', param) + logger_debug( + ' validate_option_dependencies: skip param:', param) continue value = values.get(param.name) if TRACE: - logger_debug(' validate_option_dependencies: param:', param, 'value:', value) + logger_debug(' validate_option_dependencies: param:', + param, 'value:', value) param.validate_dependencies(ctx, value) @@ -534,7 +538,7 @@ def _is_set(_value, _param): if TRACE: logger_debug() logger_debug('Checking param:', param) - logger_debug(' value:', value, 'is_set:' , is_set) + logger_debug(' value:', value, 'is_set:', is_set) if not is_set: return @@ -582,7 +586,7 @@ def _is_set(_value, _param): if TRACE: logger_debug(' Checking oparam:', oparam) - logger_debug(' value:', ovalue, 'ois_set:' , ois_set) + logger_debug(' value:', ovalue, 'ois_set:', ois_set) # by convention the last opt is the long form oopt = oparam.opts[-1] @@ -594,7 +598,7 @@ def _is_set(_value, _param): 'You must set all of these options if you use this option.' % locals()) raise click.UsageError(msg) - if not required and ois_set: + if not required and ois_set: msg = ('The option %(opt)s cannot be used together with the %(oopts)s option(s) ' 'and %(oopt)s is used. ' 'You can set only one of these options at a time.' % locals()) diff --git a/src/commoncode/codec.py b/src/commoncode/codec.py index 6a0f11af..07070680 100644 --- a/src/commoncode/codec.py +++ b/src/commoncode/codec.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -13,8 +13,9 @@ Numbers to bytes or strings and URLs coder/decoders. """ -c2i = lambda c: c -i2c = lambda i: bytes([i]) + +def c2i(c): return c +def i2c(i): return bytes([i]) def num_to_bin(num): diff --git a/src/commoncode/command.py b/src/commoncode/command.py index 65818ea3..053e7fb1 100644 --- a/src/commoncode/command.py +++ b/src/commoncode/command.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -170,12 +170,15 @@ def get_env(base_vars=None, lib_dir=None): new_path = f'{lib_dir}' # on Linux/posix ld_lib_path = os.environ.get(LD_LIBRARY_PATH) - env_vars.update({LD_LIBRARY_PATH: update_path_var(ld_lib_path, new_path)}) + env_vars.update( + {LD_LIBRARY_PATH: update_path_var(ld_lib_path, new_path)}) # on Mac, though LD_LIBRARY_PATH should work too dyld_lib_path = os.environ.get(DYLD_LIBRARY_PATH) - env_vars.update({DYLD_LIBRARY_PATH: update_path_var(dyld_lib_path, new_path)}) + env_vars.update( + {DYLD_LIBRARY_PATH: update_path_var(dyld_lib_path, new_path)}) - env_vars = {text.as_unicode(k): text.as_unicode(v) for k, v in env_vars.items()} + env_vars = {text.as_unicode(k): text.as_unicode(v) + for k, v in env_vars.items()} return env_vars @@ -216,7 +219,8 @@ def load_shared_library(dll_loc, *args): Return the loaded shared library object from the ``dll_loc`` location. """ if not dll_loc or not path.exists(dll_loc): - raise ImportError(f'Shared library does not exists: dll_loc: {dll_loc}') + raise ImportError( + f'Shared library does not exists: dll_loc: {dll_loc}') if not isinstance(dll_loc, str): dll_loc = os.fsdecode(dll_loc) diff --git a/src/commoncode/compat.py b/src/commoncode/compat.py index 7c059799..001464c7 100644 --- a/src/commoncode/compat.py +++ b/src/commoncode/compat.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/src/commoncode/datautils.py b/src/commoncode/datautils.py index 1b848d54..ea71f9aa 100644 --- a/src/commoncode/datautils.py +++ b/src/commoncode/datautils.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/src/commoncode/date.py b/src/commoncode/date.py index 62dd6937..81da3d0d 100644 --- a/src/commoncode/date.py +++ b/src/commoncode/date.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -38,4 +38,4 @@ def secs_from_epoch(d): """ # FIXME: what does this do? return calendar.timegm(datetime.strptime(d.split('.')[0], - '%Y-%m-%d %H:%M:%S').timetuple()) + '%Y-%m-%d %H:%M:%S').timetuple()) diff --git a/src/commoncode/distro.py b/src/commoncode/distro.py index 207bfba3..3b22354d 100644 --- a/src/commoncode/distro.py +++ b/src/commoncode/distro.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/src/commoncode/fetch.py b/src/commoncode/fetch.py index 43e47fed..9428e32f 100644 --- a/src/commoncode/fetch.py +++ b/src/commoncode/fetch.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -40,7 +40,8 @@ def download_url(url, file_name=None, verify=True, timeout=10): status = response.status_code if status != 200: - msg = 'download_url: Download failed for %(url)r with %(status)r' % locals() + msg = 'download_url: Download failed for %(url)r with %(status)r' % locals( + ) logger.error(msg) raise Exception(msg) diff --git a/src/commoncode/fileset.py b/src/commoncode/fileset.py index dbd4c1c8..99323f83 100644 --- a/src/commoncode/fileset.py +++ b/src/commoncode/fileset.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -83,14 +83,16 @@ def is_included(path, includes=None, excludes=None): if includes: included = get_matches(path, includes, all_matches=False) if TRACE: - logger.debug('in_fileset: path: %(path)r included:%(included)r' % locals()) + logger.debug( + 'in_fileset: path: %(path)r included:%(included)r' % locals()) if not included: return False if excludes: excluded = get_matches(path, excludes, all_matches=False) if TRACE: - logger.debug('in_fileset: path: %(path)r excluded:%(excluded)r .' % locals()) + logger.debug( + 'in_fileset: path: %(path)r excluded:%(excluded)r .' % locals()) if excluded: return False @@ -115,11 +117,13 @@ def get_matches(path, patterns, all_matches=False): segments = paths.split(pathstripped) if TRACE: - logger.debug('_match: path: %(path)r patterns:%(patterns)r.' % locals()) + logger.debug( + '_match: path: %(path)r patterns:%(patterns)r.' % locals()) matches = [] if not isinstance(patterns, dict): - assert isinstance(patterns, (list, tuple)), 'Invalid patterns: {}'.format(patterns) + assert isinstance(patterns, (list, tuple) + ), 'Invalid patterns: {}'.format(patterns) patterns = {p: p for p in patterns} for pat, value in patterns.items(): diff --git a/src/commoncode/filetype.py b/src/commoncode/filetype.py index 3f2201b8..bc8cc7b1 100644 --- a/src/commoncode/filetype.py +++ b/src/commoncode/filetype.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/src/commoncode/fileutils.py b/src/commoncode/fileutils.py index 584b7ccf..d0aa38b5 100644 --- a/src/commoncode/fileutils.py +++ b/src/commoncode/fileutils.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 AND Python-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -321,7 +321,7 @@ def walk(location, ignored=None, follow_symlinks=False): logger_debug('walk: ignored:', location, is_ignored) return - if filetype.is_file(location, follow_symlinks=follow_symlinks) : + if filetype.is_file(location, follow_symlinks=follow_symlinks): yield parent_directory(location), [], [file_name(location)] elif filetype.is_dir(location, follow_symlinks=follow_symlinks): diff --git a/src/commoncode/functional.py b/src/commoncode/functional.py index 40e275d7..8bb6f64f 100644 --- a/src/commoncode/functional.py +++ b/src/commoncode/functional.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/src/commoncode/hash.py b/src/commoncode/hash.py index 231ebe1e..61be3758 100644 --- a/src/commoncode/hash.py +++ b/src/commoncode/hash.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/src/commoncode/ignore.py b/src/commoncode/ignore.py index 881b93f5..dc21d678 100644 --- a/src/commoncode/ignore.py +++ b/src/commoncode/ignore.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -164,14 +164,14 @@ def get_ignores(location, include_defaults=True): ignores_VCS = { '.bzr': 'Default ignore: Bazaar artifact', - '.bzrignore' : 'Default ignore: Bazaar config artifact', + '.bzrignore': 'Default ignore: Bazaar config artifact', '.git': 'Default ignore: Git artifact', - '.gitignore' : 'Default ignore: Git config artifact', + '.gitignore': 'Default ignore: Git config artifact', '.gitattributes': 'Default ignore: Git config artifact', '.hg': 'Default ignore: Mercurial artifact', - '.hgignore' : 'Default ignore: Mercurial config artifact', + '.hgignore': 'Default ignore: Mercurial config artifact', '.repo': 'Default ignore: Multiple Git repository artifact', @@ -226,8 +226,8 @@ def get_ignores(location, include_defaults=True): } ignores_CI = { - '.travis.yml' : 'Default ignore: Travis config', - '.coveragerc' : 'Default ignore: Coverall config', + '.travis.yml': 'Default ignore: Travis config', + '.coveragerc': 'Default ignore: Coverall config', } ignores_Python = { diff --git a/src/commoncode/paths.py b/src/commoncode/paths.py index e9e71410..8718c503 100644 --- a/src/commoncode/paths.py +++ b/src/commoncode/paths.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -157,7 +157,8 @@ def resolve(path, posix=True): posix_illegal_chars_re = r'[^' + posix_legal_chars + r']' posix_illegal_chars_exc_spaces_re = r'[^' + posix_legal_chars_inc_spaces + r']' replace_illegal_posix_chars = re.compile(posix_illegal_chars_re).sub -replace_illegal_posix_chars_exc_spaces = re.compile(posix_illegal_chars_exc_spaces_re).sub +replace_illegal_posix_chars_exc_spaces = re.compile( + posix_illegal_chars_exc_spaces_re).sub ILLEGAL_WINDOWS_NAMES = set([ diff --git a/src/commoncode/resource.py b/src/commoncode/resource.py index 7a9d1550..2e799481 100644 --- a/src/commoncode/resource.py +++ b/src/commoncode/resource.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -159,11 +159,13 @@ class Header(object): tool_name = String(help='Name of the tool used such as scancode-toolkit.') tool_version = String(default='', help='Tool version used such as v1.2.3.') - options = Mapping(help='Mapping of key/values describing the options used with this tool.') + options = Mapping( + help='Mapping of key/values describing the options used with this tool.') notice = String(default='', help='Notice text for this tool.') start_timestamp = String(help='Start timestamp for this header.') end_timestamp = String(help='End timestamp for this header.') - output_format_version = String(help='Version for the output data format, such as v1.1 .') + output_format_version = String( + help='Version for the output data format, such as v1.1 .') duration = String(help='Scan duration in seconds.') message = String(help='Message text.') errors = List(help='List of error messages.') @@ -292,7 +294,8 @@ def __init__( self.is_file = filetype_is_file(location) # True if this codebase root is a file or an empty directory. - self.has_single_resource = bool(self.is_file or not os.listdir(location)) + self.has_single_resource = bool( + self.is_file or not os.listdir(location)) ######################################################################## # Set up caching, summary, timing, and error info @@ -309,7 +312,7 @@ def _prepare_clean_paths(self, paths=tuple()): """ paths = (clean_path(p) for p in (paths or []) if p) # we sort by path segments (e.g. essentially a topo sort) - _sorter = lambda p: p.split('/') + def _sorter(p): return p.split('/') return sorted(paths, key=_sorter) def _setup_essentials(self, temp_dir=temp_dir, max_in_memory=10000): @@ -438,7 +441,8 @@ def _populate(self): # Codebase attributes to use. Configured with plugin attributes if # present. self.codebase_attributes = self._collect_codebase_attributes() - cbac = _CodebaseAttributes.from_attributes(attributes=self.codebase_attributes) + cbac = _CodebaseAttributes.from_attributes( + attributes=self.codebase_attributes) self.attributes = cbac() # Resource sub-class to use. Configured with plugin attributes if present @@ -487,7 +491,8 @@ def _create_resources_from_paths(self, root, paths): # create all parents. The last parent is the one we want to use parent = root if TRACE: - logger_debug('Codebase._create_resources_from_paths: parent', parent) + logger_debug( + 'Codebase._create_resources_from_paths: parent', parent) for parent_path in get_ancestor_paths(path, include_self=False): if TRACE: logger_debug( @@ -497,7 +502,8 @@ def _create_resources_from_paths(self, root, paths): continue newpar = parents_by_path.get(parent_path) if TRACE: - logger_debug(' Codebase._create_resources_from_paths: newpar', repr(newpar)) + logger_debug( + ' Codebase._create_resources_from_paths: newpar', repr(newpar)) if not newpar: newpar = self._get_or_create_resource( @@ -527,7 +533,8 @@ def _create_resources_from_paths(self, root, paths): is_file=isfile(res_loc), ) if TRACE: - logger_debug('Codebase._create_resources_from_paths: resource', res) + logger_debug( + 'Codebase._create_resources_from_paths: resource', res) def _create_resources_from_root(self, root): # without paths we walks the root location top-down @@ -588,7 +595,8 @@ def _create_root_resource(self): """ # we cannot recreate a root if it exists!! if self.root: - raise TypeError('Root resource already exists and cannot be recreated') + raise TypeError( + 'Root resource already exists and cannot be recreated') location = self.location name = file_name(location) @@ -597,7 +605,8 @@ def _create_root_resource(self): path = Resource.build_path(root_location=location, location=location) if TRACE: - logger_debug(f' Codebase._create_root_resource: {path} is_file: {self.is_file}') + logger_debug( + f' Codebase._create_root_resource: {path} is_file: {self.is_file}') logger_debug() root = self.resource_class( @@ -638,11 +647,13 @@ def _get_or_create_resource( existing = self.get_resource(path) if existing: if TRACE: - logger_debug(' Codebase._get_or_create_resource: path already exists:', path) + logger_debug( + ' Codebase._get_or_create_resource: path already exists:', path) return existing if self._use_disk_cache_for_resource(): - cache_location = self._get_resource_cache_location(path=path, create_dirs=True) + cache_location = self._get_resource_cache_location( + path=path, create_dirs=True) else: cache_location = None @@ -752,18 +763,20 @@ def _exists_on_disk(self, path): """ path = clean_path(path) if not self._exists_in_memory(path): - cache_location = self._get_resource_cache_location(path, create_dirs=False) + cache_location = self._get_resource_cache_location( + path, create_dirs=False) if cache_location: return exists(cache_location) - ########### FIXME: the PATH SHOULD NOT INCLUDE THE ROOT NAME + # FIXME: the PATH SHOULD NOT INCLUDE THE ROOT NAME def get_resource(self, path): """ Return the Resource with `path` or None if it does not exists. The ``path`` must be relative to the root (and including the root name as its first segment). """ - assert isinstance(path, str), f'Invalid path: {path!r} is not a string.' + assert isinstance( + path, str), f'Invalid path: {path!r} is not a string.' path = clean_path(path) if TRACE: msg = [' Codebase.get_resource:', 'path:', path] @@ -788,7 +801,8 @@ def get_resource(self, path): pass else: # this should never happen - raise Exception(f'get_resource: Internal error when getting {path!r}') + raise Exception( + f'get_resource: Internal error when getting {path!r}') if TRACE: logger_debug(' Resource:', res) @@ -830,7 +844,8 @@ def _dump_resource(self, resource): # TODO: consider messagepack or protobuf for compact/faster processing? with open(cache_location, 'w') as cached: - cached.write(json.dumps(resource.serialize(), check_circular=False)) + cached.write(json.dumps( + resource.serialize(), check_circular=False)) # TODO: consider adding a small LRU cache in front of this for perf? def _load_resource(self, path): @@ -838,7 +853,8 @@ def _load_resource(self, path): Return a Resource with ``path`` loaded from the disk cache. """ path = clean_path(path) - cache_location = self._get_resource_cache_location(path, create_dirs=False) + cache_location = self._get_resource_cache_location( + path, create_dirs=False) if TRACE: logger_debug( @@ -849,7 +865,8 @@ def _load_resource(self, path): ) if not exists(cache_location): - raise ResourceNotInCache(f'Failed to load Resource: {path} from {cache_location!r}') + raise ResourceNotInCache( + f'Failed to load Resource: {path} from {cache_location!r}') # TODO: consider messagepack or protobuf for compact/faster processing try: @@ -863,7 +880,8 @@ def _load_resource(self, path): cached_data = cached.read() msg = ( f'ERROR: failed to load resource from cached location: {cache_location} ' - 'with content:\n\n' + repr(cached_data) + '\n\n' + traceback.format_exc() + 'with content:\n\n' + + repr(cached_data) + '\n\n' + traceback.format_exc() ) raise Exception(msg) from e @@ -873,7 +891,8 @@ def _remove_resource(self, resource): Does not remove children. """ if resource.is_root: - raise TypeError(f'Cannot remove the root resource from codebase: {resource!r}') + raise TypeError( + f'Cannot remove the root resource from codebase: {resource!r}') # remove from in-memory cache. The disk cache is cleared on exit. self.resources_by_path.pop(resource.path, None) @@ -890,7 +909,8 @@ def remove_resource(self, resource): logger_debug(' resource', resource) if resource.is_root: - raise TypeError(f'Cannot remove the root resource from codebase: {resource!r}') + raise TypeError( + f'Cannot remove the root resource from codebase: {resource!r}') removed_paths = set() @@ -1387,9 +1407,10 @@ def children(self, codebase, names=()): child_path = partial(posixpath_join, self.path) get_child = codebase.get_resource - children = [get_child(path=child_path(name)) for name in children_names] + children = [get_child(path=child_path(name)) + for name in children_names] - _sorter = lambda r: (r.has_children(), r.name.lower(), r.name) + def _sorter(r): return (r.has_children(), r.name.lower(), r.name) return sorted((c for c in children if c), key=_sorter) def has_parent(self): @@ -1504,10 +1525,11 @@ def to_dict( # this will catch every attribute that has been added dynamically, such # as scan-provided resource_attributes - other_data = attr.asdict(self, filter=self_fields_filter, dict_factory=dict) + other_data = attr.asdict( + self, filter=self_fields_filter, dict_factory=dict) # FIXME: make a deep copy of the data first!!!! - # see https://github.com/nexB/scancode-toolkit/issues/1199 + # see https://github.com/aboutcode-org/scancode-toolkit/issues/1199 res.update(other_data) if with_timing: @@ -1720,7 +1742,8 @@ def _get_scan_data(self, location): files = scan_data.get('files') if files: for f in files: - f['path'] = posixpath_join(f'codebase-{idx}', clean_path(f['path'])) + f['path'] = posixpath_join( + f'codebase-{idx}', clean_path(f['path'])) combined_scan_data['files'].extend(files) else: raise Exception( @@ -1833,7 +1856,8 @@ def _populate(self, scan_data): # Codebase attributes to use. Configured with scan_data and plugin # attributes if present. self.codebase_attributes = self._collect_codebase_attributes(scan_data) - cbac = _CodebaseAttributes.from_attributes(attributes=self.codebase_attributes) + cbac = _CodebaseAttributes.from_attributes( + attributes=self.codebase_attributes) self.attributes = cbac() # now populate top level codebase attributes @@ -2053,12 +2077,14 @@ def _create_root_resource(self, name, path, is_file): """ # we cannot recreate a root if it exists!! if self.root: - raise TypeError('Root resource already exists and cannot be recreated') + raise TypeError( + 'Root resource already exists and cannot be recreated') path = clean_path(path) if TRACE: - logger_debug(f' VirtualCodebase._create_root_resource: {path!r} is_file: {is_file}') + logger_debug( + f' VirtualCodebase._create_root_resource: {path!r} is_file: {is_file}') root = self.resource_class( name=name, @@ -2076,7 +2102,8 @@ def _create_root_resource(self, name, path, is_file): return root -KNOW_PROPS = set(['type', 'base_name', 'extension', 'path', 'name', 'path_segments']) +KNOW_PROPS = set(['type', 'base_name', 'extension', + 'path', 'name', 'path_segments']) def remove_properties_and_basics(resource_data): diff --git a/src/commoncode/system.py b/src/commoncode/system.py index 07938f9b..f69fe9df 100644 --- a/src/commoncode/system.py +++ b/src/commoncode/system.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -94,6 +94,7 @@ def is_on_ubuntu_22(os_release_location='/etc/os-release'): os_release_info = parse_os_release(os_release_location) return os_release_info['ID'] == 'ubuntu' and '22' in os_release_info['VERSION_ID'] + on_ubuntu_22 = is_on_ubuntu_22() del is_on_ubuntu_22 diff --git a/src/commoncode/testcase.py b/src/commoncode/testcase.py index 2fd9fd31..3fe61445 100644 --- a/src/commoncode/testcase.py +++ b/src/commoncode/testcase.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -138,7 +138,7 @@ def get_temp_file(self, extension=None, dir_name='td', file_name='tf'): extension = '.txt' if extension and not extension.startswith('.'): - extension = '.' + extension + extension = '.' + extension file_name = file_name + extension temp_dir = self.get_temp_dir(dir_name) @@ -167,7 +167,8 @@ def get_temp_dir(self, sub_dir_path=None): if sub_dir_path: # create a sub directory hierarchy if requested sub_dir_path = to_os_native_path(sub_dir_path) - test_run_temp_subdir = path.join(test_run_temp_subdir, sub_dir_path) + test_run_temp_subdir = path.join( + test_run_temp_subdir, sub_dir_path) fileutils.create_dir(test_run_temp_subdir) return test_run_temp_subdir @@ -237,7 +238,8 @@ def phase3(self): Find out differences between common files. Ensure we are using content comparison, not os.stat-only. """ - comp = filecmp.cmpfiles(self.left, self.right, self.common_files, shallow=False) + comp = filecmp.cmpfiles(self.left, self.right, + self.common_files, shallow=False) self.same_files, self.diff_files, self.funny_files = comp @@ -248,7 +250,7 @@ def is_same(dir1, dir2): """ compared = dircmp(dir1, dir2) if (compared.left_only or compared.right_only or compared.diff_files - or compared.funny_files): + or compared.funny_files): return False for subdir in compared.common_dirs: @@ -347,8 +349,10 @@ def get_test_file_pairs(test_dir): # ensure that we haev no dangling files if dangling_test_files or dangling_data_files: - msg = ['Dangling missing test files without a YAML data file:'] + sorted(dangling_test_files) - msg += ['Dangling missing YAML data files without a test file'] + sorted(dangling_data_files) + msg = ['Dangling missing test files without a YAML data file:'] + \ + sorted(dangling_test_files) + msg += ['Dangling missing YAML data files without a test file'] + \ + sorted(dangling_data_files) msg = '\n'.join(msg) print(msg) raise Exception(msg) @@ -370,7 +374,8 @@ def get_test_file_pairs(test_dir): dupes = list(chain.from_iterable( paths for paths in paths_ignoring_case.values() if len(paths) != 1)) if dupes: - msg = ['Non unique test/data file(s) found when ignoring case!'] + sorted(dupes) + msg = [ + 'Non unique test/data file(s) found when ignoring case!'] + sorted(dupes) msg = '\n'.join(msg) print(msg) diff --git a/src/commoncode/text.py b/src/commoncode/text.py index df2f3463..b2f4ea1c 100644 --- a/src/commoncode/text.py +++ b/src/commoncode/text.py @@ -3,7 +3,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/src/commoncode/timeutils.py b/src/commoncode/timeutils.py index 44b68543..0bcb0116 100644 --- a/src/commoncode/timeutils.py +++ b/src/commoncode/timeutils.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/src/commoncode/urn.py b/src/commoncode/urn.py index fcda69a6..37d9fa9c 100644 --- a/src/commoncode/urn.py +++ b/src/commoncode/urn.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/src/commoncode/version.py b/src/commoncode/version.py index e112994c..83cb2460 100644 --- a/src/commoncode/version.py +++ b/src/commoncode/version.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -22,19 +22,19 @@ def VERSION_PATTERNS_REGEX(): r'((\.|\-)(((alpha|dev|beta|rc|FINAL|final|pre)(\-|\_)\d+[A-Za-z]?' r'(\-RELEASE)?)|alpha|dev(\.\d+\.\d+)?' r'|beta|BETA|final|FINAL|release|fixed|(cr\d(\_\d*)?)))?', - # - r'[A-Za-z]?(\d+\_){1,3}\d+\_?[A-Za-z]{0,2}\d+', - # - r'(b|rc|r|v|RC|alpha|beta|BETA|M|m|pre|revision-)\d+(\-\d+)?', - # - r'current|previous|latest|alpha|beta', - # - r'\d{4}-\d{2}-\d{2}', - # - r'(\d(\-|\_)){1,2}\d', - # - r'\d{5,14}', - ]] + # + r'[A-Za-z]?(\d+\_){1,3}\d+\_?[A-Za-z]{0,2}\d+', + # + r'(b|rc|r|v|RC|alpha|beta|BETA|M|m|pre|revision-)\d+(\-\d+)?', + # + r'current|previous|latest|alpha|beta', + # + r'\d{4}-\d{2}-\d{2}', + # + r'(\d(\-|\_)){1,2}\d', + # + r'\d{5,14}', + ]] def hint(path): diff --git a/tests/test_cliutils.py b/tests/test_cliutils.py index c0aebe76..a1d88ce7 100644 --- a/tests/test_cliutils.py +++ b/tests/test_cliutils.py @@ -2,20 +2,19 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +from commoncode.cliutils import PluggableCommandLineOption +from commoncode.cliutils import GroupedHelpCommand +from commoncode.cliutils import fixed_width_file_name +from commoncode.testcase import FileDrivenTesting +from click.testing import CliRunner import os import click click.disable_unicode_literals_warning = True -from click.testing import CliRunner - -from commoncode.testcase import FileDrivenTesting -from commoncode.cliutils import fixed_width_file_name -from commoncode.cliutils import GroupedHelpCommand -from commoncode.cliutils import PluggableCommandLineOption class TestFixedWidthFilename(FileDrivenTesting): @@ -51,22 +50,26 @@ def test_fixed_width_file_name_without_extension(self): assert test == expected def test_fixed_width_file_name_with_posix_path_without_shortening(self): - test = fixed_width_file_name('C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/drupal.js', 25) + test = fixed_width_file_name( + 'C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/drupal.js', 25) expected = 'drupal.js' assert test == expected def test_fixed_width_file_name_with_posix_path_with_shortening(self): - test = fixed_width_file_name('C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/012345678901234567890123.c', 25) + test = fixed_width_file_name( + 'C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/012345678901234567890123.c', 25) expected = '0123456789...4567890123.c' assert test == expected def test_fixed_width_file_name_with_win_path_without_shortening(self): - test = fixed_width_file_name('C\\:Documents_and_Settings\\Boki\\Desktop\\head\\patches\\drupal6\\drupal.js', 25) + test = fixed_width_file_name( + 'C\\:Documents_and_Settings\\Boki\\Desktop\\head\\patches\\drupal6\\drupal.js', 25) expected = 'drupal.js' assert test == expected def test_fixed_width_file_name_with_win_path_with_shortening(self): - test = fixed_width_file_name('C\\:Documents_and_Settings\\Boki\\Desktop\\head\\patches\\drupal6\\012345678901234567890123.c', 25) + test = fixed_width_file_name( + 'C\\:Documents_and_Settings\\Boki\\Desktop\\head\\patches\\drupal6\\012345678901234567890123.c', 25) expected = '0123456789...4567890123.c' assert test == expected @@ -92,7 +95,7 @@ def scan(opt): result = runner.invoke(scan, ['--help']) from commoncode.cliutils import MISC_GROUP assert MISC_GROUP in result.output - assert '--opt Help text for option' in result.output + assert '--opt Help text for option' in result.output def test_GroupedHelpCommand_with_help_group_and_sort_order_with_custom_class(self): diff --git a/tests/test_cliutils_progressbar.py b/tests/test_cliutils_progressbar.py index add6cd48..1ee8dca9 100644 --- a/tests/test_cliutils_progressbar.py +++ b/tests/test_cliutils_progressbar.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -11,6 +11,7 @@ from commoncode.testcase import FileDrivenTesting from commoncode.cliutils import progressmanager + class TestProgressBar(FileDrivenTesting): test_data_dir = os.path.join(os.path.dirname(__file__), 'data') diff --git a/tests/test_codec.py b/tests/test_codec.py index 7b2041f9..e852e707 100644 --- a/tests/test_codec.py +++ b/tests/test_codec.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -56,7 +56,7 @@ def test_num_to_bin_bin_to_num_is_idempotent(self): assert result == expected def test_urlsafe_b64encode_int_zero(self): - assert urlsafe_b64encode_int(0) == b'AA==' + assert urlsafe_b64encode_int(0) == b'AA==' def test_urlsafe_b64encode_int_basic(self): assert urlsafe_b64encode_int(123123123123) == b'HKq1w7M=' diff --git a/tests/test_command.py b/tests/test_command.py index b24fc06d..1f54c73b 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -3,7 +3,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -140,7 +140,8 @@ def test_find_in_path(self): d2 = self.get_temp_dir('bar') filename = 'baz' - assert None == command.find_in_path(filename, searchable_paths=(d1, d2,)) + assert None == command.find_in_path( + filename, searchable_paths=(d1, d2,)) f2 = os.path.join(d2, filename) with open(f2, 'w') as o: diff --git a/tests/test_date.py b/tests/test_date.py index a6473593..4f30f27a 100644 --- a/tests/test_date.py +++ b/tests/test_date.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/tests/test_fileset.py b/tests/test_fileset.py index b72e17e5..79e44a8f 100644 --- a/tests/test_fileset.py +++ b/tests/test_fileset.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -27,7 +27,7 @@ def test_is_included_basic(self): def test_is_included_in_fileset(self): incs = {'/common/src/*': '.scanignore'} - excs = {'/common/src/*.so':'.scanignore'} + excs = {'/common/src/*.so': '.scanignore'} assert not fileset.is_included(None, incs, excs) assert not fileset.is_included('', incs, excs) assert not fileset.is_included('/', incs, excs) @@ -46,12 +46,12 @@ def test_is_included_in_fileset_2(self): def test_is_included_is_included_exclusions(self): incs = {'/src/*': '.scanignore'} - excs = {'/src/*.so':'.scanignore'} + excs = {'/src/*.so': '.scanignore'} assert not fileset.is_included('/src/dist/build/mylib.so', incs, excs) def test_is_included_is_included_exclusions_2(self): incs = {'src': '.scanignore'} - excs = {'src/*.so':'.scanignore'} + excs = {'src/*.so': '.scanignore'} assert fileset.is_included('/some/src/this/that', incs, excs) assert not fileset.is_included('/src/dist/build/mylib.so', incs, excs) diff --git a/tests/test_filetype.py b/tests/test_filetype.py index 2ca4d1d5..378313a0 100644 --- a/tests/test_filetype.py +++ b/tests/test_filetype.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -57,14 +57,14 @@ def test_get_type(self): # symlinks and special files are not supported on win if on_posix: - expected += [ ('2-SYMTYPE', 'l'), ('6-FIFOTYPE', 's'), ] + expected += [('2-SYMTYPE', 'l'), ('6-FIFOTYPE', 's'), ] try: assert sorted(results) == sorted(expected) except Exception as e: if on_windows and py3: # On some Windows symlinkes are detected OK (Windows 10?) but not in Windows 7 - expected += [ ('2-SYMTYPE', 'l') ] + expected += [('2-SYMTYPE', 'l')] assert sorted(results) == sorted(expected) else: raise e diff --git a/tests/test_fileutils.py b/tests/test_fileutils.py index f6e3b744..049c1c71 100644 --- a/tests/test_fileutils.py +++ b/tests/test_fileutils.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -301,7 +301,8 @@ def test_os_walk_with_unicode_path(self): def test_fileutils_walk(self): test_dir = self.get_test_loc('fileutils/walk') base = self.get_test_loc('fileutils') - result = [(as_posixpath(t.replace(base, '')), d, sorted(f),) for t, d, f in fileutils.walk(test_dir)] + result = [(as_posixpath(t.replace(base, '')), d, sorted(f),) + for t, d, f in fileutils.walk(test_dir)] expected = [ ('/walk', ['d1'], ['f', 'unicode.zip']), ('/walk/d1', ['d2'], ['f1']), @@ -336,7 +337,8 @@ def test_fileutils_walk_can_walk_an_empty_dir(self): @skipIf(on_macos_14_or_higher, 'Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635') def test_walk_can_walk_non_utf8_path_from_unicode_path(self): - test_dir = self.extract_test_tar_raw('fileutils/walk_non_utf8/non_unicode.tgz') + test_dir = self.extract_test_tar_raw( + 'fileutils/walk_non_utf8/non_unicode.tgz') test_dir = join(test_dir, 'non_unicode') if not on_linux: @@ -347,7 +349,8 @@ def test_walk_can_walk_non_utf8_path_from_unicode_path(self): @skipIf(on_macos_14_or_higher, 'Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635') def test_os_walk_can_walk_non_utf8_path_from_unicode_path(self): - test_dir = self.extract_test_tar_raw('fileutils/walk_non_utf8/non_unicode.tgz') + test_dir = self.extract_test_tar_raw( + 'fileutils/walk_non_utf8/non_unicode.tgz') test_dir = join(test_dir, 'non_unicode') result = list(os.walk(test_dir))[0] @@ -361,7 +364,8 @@ def test_walk_on_symlinks(self): test_link = join(temp_dir, 'test-dir-link') os.symlink(test_dir, test_link) results = list(fileutils.walk(test_link, follow_symlinks=True)) - results = [(os.path.basename(top), dirs, files) for top, dirs, files in results] + results = [(os.path.basename(top), dirs, files) + for top, dirs, files in results] expected = [ ('test-dir-link', ['dir'], ['a']), ('dir', [], ['b']) @@ -375,7 +379,8 @@ class TestFileUtilsIter(FileBasedTesting): def test_resource_iter(self): test_dir = self.get_test_loc('fileutils/walk') base = self.get_test_loc('fileutils') - result = [as_posixpath(f.replace(base, '')) for f in fileutils.resource_iter(test_dir, with_dirs=False)] + result = [as_posixpath(f.replace(base, '')) + for f in fileutils.resource_iter(test_dir, with_dirs=False)] expected = [ '/walk/f', '/walk/unicode.zip', @@ -387,13 +392,15 @@ def test_resource_iter(self): def test_resource_iter_can_iterate_a_single_file(self): test_file = self.get_test_loc('fileutils/walk/f') - result = [as_posixpath(f) for f in fileutils.resource_iter(test_file, with_dirs=False)] + result = [as_posixpath(f) for f in fileutils.resource_iter( + test_file, with_dirs=False)] expected = [as_posixpath(test_file)] assert result == expected def test_resource_iter_can_iterate_a_single_file_with_dirs(self): test_file = self.get_test_loc('fileutils/walk/f') - result = [as_posixpath(f) for f in fileutils.resource_iter(test_file, with_dirs=True)] + result = [as_posixpath(f) for f in fileutils.resource_iter( + test_file, with_dirs=True)] expected = [as_posixpath(test_file)] assert result == expected @@ -482,7 +489,8 @@ def test_resource_iter_can_walk_unicode_path_with_zip(self): test_dir = str(test_dir) - result = sorted([p.replace(test_dir, '') for p in fileutils.resource_iter(test_dir)]) + result = sorted([p.replace(test_dir, '') + for p in fileutils.resource_iter(test_dir)]) if on_linux: expected = [ u'/2.csv', @@ -505,7 +513,8 @@ def test_resource_iter_can_walk_unicode_path_with_zip(self): @skipIf(on_macos_14_or_higher, 'Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635') def test_resource_iter_can_walk_non_utf8_path_from_unicode_path_with_dirs(self): - test_dir = self.extract_test_tar_raw('fileutils/walk_non_utf8/non_unicode.tgz') + test_dir = self.extract_test_tar_raw( + 'fileutils/walk_non_utf8/non_unicode.tgz') test_dir = join(test_dir, 'non_unicode') result = list(fileutils.resource_iter(test_dir, with_dirs=True)) @@ -513,7 +522,8 @@ def test_resource_iter_can_walk_non_utf8_path_from_unicode_path_with_dirs(self): @skipIf(on_macos_14_or_higher, 'Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635') def test_resource_iter_can_walk_non_utf8_path_from_unicode_path(self): - test_dir = self.extract_test_tar_raw('fileutils/walk_non_utf8/non_unicode.tgz') + test_dir = self.extract_test_tar_raw( + 'fileutils/walk_non_utf8/non_unicode.tgz') test_dir = join(test_dir, 'non_unicode') result = list(fileutils.resource_iter(test_dir, with_dirs=False)) @@ -525,7 +535,8 @@ def test_resource_iter_follow_symlinks(self): temp_dir = fileutils.get_temp_dir() test_link = join(temp_dir, 'test-dir-link') os.symlink(test_dir, test_link) - result = [os.path.basename(f) for f in fileutils.resource_iter(test_dir, follow_symlinks=True)] + result = [os.path.basename(f) for f in fileutils.resource_iter( + test_dir, follow_symlinks=True)] expected = [ 'dir', 'a', @@ -546,7 +557,7 @@ def test_file_base_name_on_path_and_location_1(self): result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name - def test_file_base_name_on_file_path_for_dot_file (self): + def test_file_base_name_on_file_path_for_dot_file(self): test_dir = self.get_test_loc('fileutils/basename') test_file = 'a/.a/' expected_name = '.a' @@ -876,7 +887,8 @@ def test_parent_directory_on_path_and_location_1(self): result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory((os.path.join(test_dir, test_file))) + result = fileutils.parent_directory( + (os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) @@ -888,7 +900,8 @@ def test_parent_directory_on_path_and_location_2(self): result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory((os.path.join(test_dir, test_file))) + result = fileutils.parent_directory( + (os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) @@ -900,7 +913,8 @@ def test_parent_directory_on_path_and_location_3(self): result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory((os.path.join(test_dir, test_file))) + result = fileutils.parent_directory( + (os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) @@ -912,7 +926,8 @@ def test_parent_directory_on_path_and_location_4(self): result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory((os.path.join(test_dir, test_file))) + result = fileutils.parent_directory( + (os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) @@ -924,7 +939,8 @@ def test_parent_directory_on_path_and_location_5(self): result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory((os.path.join(test_dir, test_file))) + result = fileutils.parent_directory( + (os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) @@ -936,7 +952,8 @@ def test_parent_directory_on_path_and_location_6(self): result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory((os.path.join(test_dir, test_file))) + result = fileutils.parent_directory( + (os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) @@ -948,7 +965,8 @@ def test_parent_directory_on_path_and_location_7(self): result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory((os.path.join(test_dir, test_file))) + result = fileutils.parent_directory( + (os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) @@ -960,7 +978,8 @@ def test_parent_directory_on_path_and_location_8(self): result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory((os.path.join(test_dir, test_file))) + result = fileutils.parent_directory( + (os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) @@ -972,7 +991,8 @@ def test_parent_directory_on_path_and_location_9(self): result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory((os.path.join(test_dir, test_file))) + result = fileutils.parent_directory( + (os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) @@ -984,6 +1004,7 @@ def test_parent_directory_on_path_and_location_10(self): result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory((os.path.join(test_dir, test_file))) + result = fileutils.parent_directory( + (os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) diff --git a/tests/test_functional.py b/tests/test_functional.py index 5f9d5728..0f4b76c9 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/tests/test_hash.py b/tests/test_hash.py index 4c1dc03d..0ab15b52 100644 --- a/tests/test_hash.py +++ b/tests/test_hash.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -75,15 +75,18 @@ def test_sha1_checksum(self): def test_sha256_checksum(self): test_file = self.get_test_loc('hash/dir1/a.png') - assert u'1b598db6fee8f1ec7bb919c0adf68956f3d20af8c9934a9cf2db52e1347efd35' == sha256(test_file) + assert u'1b598db6fee8f1ec7bb919c0adf68956f3d20af8c9934a9cf2db52e1347efd35' == sha256( + test_file) def test_sha512_checksum(self): test_file = self.get_test_loc('hash/dir1/a.png') - assert u'5be9e01cd20ff288fd3c3fc46be5c2747eaa2c526197125330947a95cdb418222176b182a4680f0e435ba8f114363c45a67b30eed9a9222407e63ccbde46d3b4' == sha512(test_file) + assert u'5be9e01cd20ff288fd3c3fc46be5c2747eaa2c526197125330947a95cdb418222176b182a4680f0e435ba8f114363c45a67b30eed9a9222407e63ccbde46d3b4' == sha512( + test_file) def test_checksum_sha1(self): test_file = self.get_test_loc('hash/dir1/a.txt') - assert '3ca69e8d6c234a469d16ac28a4a658c92267c423' == checksum(test_file, 'sha1') + assert '3ca69e8d6c234a469d16ac28a4a658c92267c423' == checksum( + test_file, 'sha1') def test_checksum_md5(self): test_file = self.get_test_loc('hash/dir1/a.txt') diff --git a/tests/test_ignore.py b/tests/test_ignore.py index 22eceecf..984c512c 100644 --- a/tests/test_ignore.py +++ b/tests/test_ignore.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -59,7 +59,8 @@ def test_is_ignored_default_ignores_mac2(self): test_dir = self.extract_test_tar('ignore/excludes/mac.tgz') test_base = os.path.join(test_dir, 'mac') - test = os.path.join(test_base, '__MACOSX/comp_match/smallrepo/._jetty_1.0_index.csv') + test = os.path.join( + test_base, '__MACOSX/comp_match/smallrepo/._jetty_1.0_index.csv') assert ignore.is_ignored(test, ignore.default_ignores, {}) def test_is_ignored_default_ignores_mac3(self): @@ -140,8 +141,8 @@ def test_fileset_is_included_with_default_ignore_does_not_skip_one_char_names(se from commoncode import fileset tests = [c for c in 'HFS+ Private Data'] + 'HFS+ Private Data'.split() result = [(t, - fileset.is_included(t, excludes=ignore.default_ignores, includes={})) - for t in tests] + fileset.is_included(t, excludes=ignore.default_ignores, includes={})) + for t in tests] expected = [ ('H', True), ('F', True), @@ -170,27 +171,37 @@ def test_fileset_is_included_with_default_ignore_does_not_skip_one_char_names(se @skipIf(on_mac or on_windows, 'We are only testing on posix for now') def test_is_ignored_path_string_skip_special(self): test_path = '/test/path' - assert ignore.is_ignored(test_path, {'asdf': 'skip'}, {}, skip_special=True) - assert not ignore.is_ignored(test_path, {'asdf': 'skip'}, {}, skip_special=False) + assert ignore.is_ignored( + test_path, {'asdf': 'skip'}, {}, skip_special=True) + assert not ignore.is_ignored( + test_path, {'asdf': 'skip'}, {}, skip_special=False) @skipIf(on_mac or on_windows, 'We are only testing on posix for now') def test_is_ignored_special_files_skip_special(self): test_fifo = self.get_temp_file() os.mkfifo(test_fifo) - assert ignore.is_ignored(test_fifo, {'asdf': 'skip'}, {}, skip_special=True) - assert not ignore.is_ignored(test_fifo, {'asdf': 'skip'}, {}, skip_special=False) + assert ignore.is_ignored( + test_fifo, {'asdf': 'skip'}, {}, skip_special=True) + assert not ignore.is_ignored( + test_fifo, {'asdf': 'skip'}, {}, skip_special=False) test_symlink = self.get_temp_file() test_file_location = self.get_test_loc('ignore/vcs.tgz') os.symlink(test_file_location, test_symlink) - assert ignore.is_ignored(test_symlink, {'asdf': 'skip'}, {}, skip_special=True) - assert not ignore.is_ignored(test_symlink, {'asdf': 'skip'}, {}, skip_special=False) + assert ignore.is_ignored( + test_symlink, {'asdf': 'skip'}, {}, skip_special=True) + assert not ignore.is_ignored( + test_symlink, {'asdf': 'skip'}, {}, skip_special=False) @skipIf(on_mac or on_windows, 'We are only testing on posix for now') def test_is_ignored_real_location_skip_special(self): test_file_location = self.get_test_loc('ignore/vcs.tgz') - assert not ignore.is_ignored(test_file_location, {'asdf': 'skip'}, {}, skip_special=True) - assert not ignore.is_ignored(test_file_location, {'asdf': 'skip'}, {}, skip_special=False) - - assert not ignore.is_ignored(test_file_location, {'asdf': 'skip'}, {}, skip_special=True) - assert not ignore.is_ignored(test_file_location, {'asdf': 'skip'}, {}, skip_special=False) + assert not ignore.is_ignored( + test_file_location, {'asdf': 'skip'}, {}, skip_special=True) + assert not ignore.is_ignored( + test_file_location, {'asdf': 'skip'}, {}, skip_special=False) + + assert not ignore.is_ignored( + test_file_location, {'asdf': 'skip'}, {}, skip_special=True) + assert not ignore.is_ignored( + test_file_location, {'asdf': 'skip'}, {}, skip_special=False) diff --git a/tests/test_paths.py b/tests/test_paths.py index be855c26..5d7d94d2 100644 --- a/tests/test_paths.py +++ b/tests/test_paths.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -14,32 +14,38 @@ class TestPortablePath(TestCase): def test_safe_path_mixed_slashes(self): - test = paths.safe_path('C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\drupal6/drupal.js') + test = paths.safe_path( + 'C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\drupal6/drupal.js') expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/drupal.js' assert test == expected def test_safe_path_mixed_slashes_and_spaces(self): - test = paths.safe_path('C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\parallel uploads/drupal.js') + test = paths.safe_path( + 'C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\parallel uploads/drupal.js') expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/parallel_uploads/drupal.js' assert test == expected def test_safe_path_windows_style(self): - test = paths.safe_path('C:\\Documents and Settings\\Administrator\\Desktop\\siftDemoV4_old\\defs.h') + test = paths.safe_path( + 'C:\\Documents and Settings\\Administrator\\Desktop\\siftDemoV4_old\\defs.h') expected = 'C/Documents_and_Settings/Administrator/Desktop/siftDemoV4_old/defs.h' assert test == expected def test_safe_path_windows_style_mixed_slashes_no_spaces(self): - test = paths.safe_path('C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\imagefield/imagefield.css') + test = paths.safe_path( + 'C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\imagefield/imagefield.css') expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/imagefield/imagefield.css' assert test == expected def test_safe_path_windows_style_spaces(self): - test = paths.safe_path('C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\js delete\\imagefield.css') + test = paths.safe_path( + 'C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\js delete\\imagefield.css') expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/js_delete/imagefield.css' assert test == expected def test_safe_path_windows_style_posix_slashes(self): - test = paths.safe_path('C:/Documents and Settings/Alex Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java') + test = paths.safe_path( + 'C:/Documents and Settings/Alex Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java') expected = 'C/Documents_and_Settings/Alex_Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java' assert test == expected @@ -79,7 +85,8 @@ def test_safe_path_windows_style_dots(self): assert test == expected def test_safe_path_windows_style_many_dots(self): - test = paths.safe_path('.\\includes\\.\\..\\..\\..\\webform.components.inc\\.') + test = paths.safe_path( + '.\\includes\\.\\..\\..\\..\\webform.components.inc\\.') expected = 'dotdot/dotdot/webform.components.inc' assert test == expected @@ -89,7 +96,8 @@ def test_safe_path_posix_style_dots(self): assert test == expected def test_safe_path_posix_style_many_dots(self): - test = paths.safe_path('./includes/./../../../../webform.components.inc/.') + test = paths.safe_path( + './includes/./../../../../webform.components.inc/.') expected = 'dotdot/dotdot/dotdot/webform.components.inc' assert test == expected @@ -117,12 +125,14 @@ def test_resolve_3(self): assert test == expected def test_resolve_4(self): - test = paths.resolve('////.//includes/./../..//..///../webform.components.inc/.') + test = paths.resolve( + '////.//includes/./../..//..///../webform.components.inc/.') expected = 'dotdot/dotdot/dotdot/webform.components.inc' assert test == expected def test_resolve_5(self): - test = paths.resolve(u'////.//includes/./../..//..///../webform.components.inc/.') + test = paths.resolve( + u'////.//includes/./../..//..///../webform.components.inc/.') expected = u'dotdot/dotdot/dotdot/webform.components.inc' assert test == expected @@ -133,11 +143,13 @@ def test_resolve_6(self): def test_portable_filename(self): expected = 'A___file__with_Spaces.mov' - assert paths.portable_filename("A:\\ file/ with Spaces.mov") == expected + assert paths.portable_filename( + "A:\\ file/ with Spaces.mov") == expected # Test `preserve_spaces` option. Spaces should not be replaced expected = 'Program Files (x86)' - assert paths.portable_filename("Program Files (x86)", preserve_spaces=True) == expected + assert paths.portable_filename( + "Program Files (x86)", preserve_spaces=True) == expected # Unresolved relative paths will be treated as a single filename. Use # resolve instead if you want to resolve paths: @@ -146,7 +158,8 @@ def test_portable_filename(self): # Unicode name are transliterated: expected = 'This_contain_UMLAUT_umlauts.txt' - assert paths.portable_filename(u'This contain UMLAUT \xfcml\xe4uts.txt') == expected + assert paths.portable_filename( + u'This contain UMLAUT \xfcml\xe4uts.txt') == expected # Check to see if illegal Windows filenames are properly handled for illegal_window_name in paths.ILLEGAL_WINDOWS_NAMES: @@ -155,13 +168,15 @@ def test_portable_filename(self): assert paths.portable_filename(illegal_window_name) == expected # Allow files with names that are illegal on Windows - assert paths.portable_filename(illegal_window_name, posix_only=True) == illegal_window_name + assert paths.portable_filename( + illegal_window_name, posix_only=True) == illegal_window_name # Check to see if the posix_only option does and does not replace # punctuation characters that are illegal in Windows filenames for valid_posix_path_char in paths.posix_legal_punctuation: test_name = f'test{valid_posix_path_char}' - assert paths.portable_filename(test_name, posix_only=True) == test_name + assert paths.portable_filename( + test_name, posix_only=True) == test_name if valid_posix_path_char not in paths.legal_punctuation: expected = f'test_' assert paths.portable_filename(test_name) == expected @@ -291,7 +306,8 @@ def test_common_path_suffix_return_None_if_no_common_suffix2(self): def test_common_path_suffix_match_only_whole_segments(self): # only segments are honored, commonality within segment is ignored - test = paths.common_path_suffix('this/is/aaaa/great/path', 'this/is/aaaaa/great/path') + test = paths.common_path_suffix( + 'this/is/aaaa/great/path', 'this/is/aaaaa/great/path') assert test == ('great/path', 2) def test_common_path_suffix_two_root(self): diff --git a/tests/test_resource.py b/tests/test_resource.py index 2d899ae9..25f2bd95 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -270,7 +270,8 @@ def test_walk_skip_root_single_file_with_children(self): codebase = Codebase(test_codebase, strip_root=True) root = codebase.root - c1 = codebase._get_or_create_resource('child1', parent=root, is_file=True) + c1 = codebase._get_or_create_resource( + 'child1', parent=root, is_file=True) codebase._get_or_create_resource('child2', parent=c1, is_file=False) results = list(codebase.walk(skip_root=True)) @@ -281,8 +282,10 @@ def test_walk_filtered_with_skip_root_and_single_file_with_children(self): test_codebase = self.get_test_loc('resource/codebase/et131x.h') codebase = Codebase(test_codebase, strip_root=True) - c1 = codebase._get_or_create_resource('some child', parent=codebase.root, is_file=True) - c2 = codebase._get_or_create_resource('some child2', parent=c1, is_file=False) + c1 = codebase._get_or_create_resource( + 'some child', parent=codebase.root, is_file=True) + c2 = codebase._get_or_create_resource( + 'some child2', parent=c1, is_file=False) c2.is_filtered = True codebase.save_resource(c2) @@ -306,7 +309,8 @@ def test_walk_skip_root_single_dir(self): def test_walk_skipped_directories_should_not_be_yielded(self): # Resources that we continue past should not be added to the result list - test_codebase = self.get_test_loc('resource/skip_directories_during_walk') + test_codebase = self.get_test_loc( + 'resource/skip_directories_during_walk') cdbs = Codebase(test_codebase) def _ignored(resource, codebase): @@ -326,7 +330,8 @@ def _ignored(resource, codebase): def test__create_resource_can_add_child_to_file(self): test_codebase = self.get_test_loc('resource/codebase/et131x.h') codebase = Codebase(test_codebase) - codebase._get_or_create_resource('some child', codebase.root, is_file=True) + codebase._get_or_create_resource( + 'some child', codebase.root, is_file=True) results = list(codebase.walk()) expected = [('et131x.h', True), ('some child', True)] assert [(r.name, r.is_file) for r in results] == expected @@ -334,7 +339,8 @@ def test__create_resource_can_add_child_to_file(self): def test__create_resource_can_add_child_to_dir(self): test_codebase = self.get_temp_dir('resource') codebase = Codebase(test_codebase) - codebase._get_or_create_resource('some child', codebase.root, is_file=False) + codebase._get_or_create_resource( + 'some child', codebase.root, is_file=False) results = list(codebase.walk()) expected = [('resource', False), ('some child', False)] assert [(r.name, r.is_file) for r in results] == expected @@ -354,7 +360,8 @@ def test_get_resource_for_multiple_resource_codebase(self): codebase = Codebase(test_codebase) assert codebase.get_resource('resource/a').path == 'resource/a' assert codebase.get_resource('/resource/c').path == 'resource/c' - assert codebase.get_resource('resource/dsasda/../b/').path == 'resource/b' + assert codebase.get_resource( + 'resource/dsasda/../b/').path == 'resource/b' def test_Resource_build_path(self): test_dir = self.get_test_loc('resource/samples') @@ -491,7 +498,8 @@ def test_distance(self): res = codebase.get_resource('dist/simple/META-INF/MANIFEST.MF') assert res.name == 'MANIFEST.MF' - assert res.full_root_path.endswith('resource/dist/simple/META-INF/MANIFEST.MF') + assert res.full_root_path.endswith( + 'resource/dist/simple/META-INF/MANIFEST.MF') assert res.distance(codebase) == 3 def test_skip_files_and_subdirs_of_ignored_dirs(self): @@ -542,7 +550,8 @@ def test_depth_walk_with_depth_2(self): 'level2_file4', 'level2_file5', ].sort() - expected_dirs = ['level1_dir1', 'level1_dir2', 'level2_dir1', 'level2_dir3'].sort() + expected_dirs = ['level1_dir1', 'level1_dir2', + 'level2_dir1', 'level2_dir3'].sort() self.assertEqual(result_dirs, expected_dirs) self.assertEqual(result_files, expected_files) @@ -652,7 +661,8 @@ def test_Codebase_with_paths_works(self): check_against_expected_json_file(results, expected_file, regen=False) def test_VirtualCodebase_with_paths_works(self): - test_codebase = self.get_test_loc('resource/with_path/virtual-codebase.json') + test_codebase = self.get_test_loc( + 'resource/with_path/virtual-codebase.json') paths = ['codebase/other dir/file'] codebase = VirtualCodebase(location=test_codebase, paths=paths) assert not codebase.errors @@ -664,7 +674,8 @@ def test_VirtualCodebase_with_paths_works(self): check_against_expected_json_file(results, expected_file, regen=False) def test_VirtualCodebase_codebase_attributes_assignment(self): - test_codebase = self.get_test_loc('resource/with_path/virtual-codebase.json') + test_codebase = self.get_test_loc( + 'resource/with_path/virtual-codebase.json') vc = VirtualCodebase( location=test_codebase, codebase_attributes=dict( @@ -691,7 +702,8 @@ def test_codebase_cache_default(self): assert not exists(cp) assert exists(parent_directory(cp)) - child = codebase._get_or_create_resource(name='child', parent=root, is_file=True) + child = codebase._get_or_create_resource( + name='child', parent=root, is_file=True) child.size = 12 codebase.save_resource(child) child_2 = codebase.get_resource(path=child.path) @@ -769,7 +781,8 @@ class TestVirtualCodebase(FileBasedTesting): test_data_dir = join(dirname(__file__), 'data') def test_virtual_codebase_walk_defaults(self): - test_file = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') + test_file = self.get_test_loc( + 'resource/virtual_codebase/virtual_codebase.json') codebase = VirtualCodebase(location=test_file) results = list(codebase.walk()) expected = [ @@ -785,7 +798,8 @@ def test_virtual_codebase_walk_defaults(self): assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_topdown(self): - test_file = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') + test_file = self.get_test_loc( + 'resource/virtual_codebase/virtual_codebase.json') codebase = VirtualCodebase(location=test_file) results = list(codebase.walk(topdown=True)) expected = [ @@ -801,7 +815,8 @@ def test_virtual_codebase_walk_topdown(self): assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_bottomup(self): - test_file = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') + test_file = self.get_test_loc( + 'resource/virtual_codebase/virtual_codebase.json') codebase = VirtualCodebase(location=test_file) results = list(codebase.walk(topdown=False)) expected = [ @@ -817,7 +832,8 @@ def test_virtual_codebase_walk_bottomup(self): assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_skip_root_basic(self): - scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/virtual_codebase.json') virtual_codebase = VirtualCodebase(location=scan_data) results = list(virtual_codebase.walk(skip_root=True)) expected = [ @@ -832,21 +848,26 @@ def test_virtual_codebase_walk_skip_root_basic(self): assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_get_path_with_strip_root_and_walk_with_skip_root(self): - scan_data = self.get_test_loc('resource/virtual_codebase/stripped-and-skipped-root.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/stripped-and-skipped-root.json') virtual_codebase = VirtualCodebase(location=scan_data) - results = [r.get_path(strip_root=True) for r in virtual_codebase.walk(skip_root=True)] + results = [r.get_path(strip_root=True) + for r in virtual_codebase.walk(skip_root=True)] expected = ['README', 'screenshot.png'] assert expected == results def test_virtual_codebase_to_list_with_strip_root_and_walk_with_skip_root(self): - scan_data = self.get_test_loc('resource/virtual_codebase/stripped-and-skipped-root.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/stripped-and-skipped-root.json') virtual_codebase = VirtualCodebase(location=scan_data) results = virtual_codebase.to_list(strip_root=True, skinny=True) - expected = [{'path': 'README', 'type': 'file'}, {'path': 'screenshot.png', 'type': 'file'}] + expected = [{'path': 'README', 'type': 'file'}, + {'path': 'screenshot.png', 'type': 'file'}] assert expected == results def test_virtual_codebase_walk_filtered_with_filtered_root(self): - scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/virtual_codebase.json') virtual_codebase = VirtualCodebase(location=scan_data) virtual_codebase.root.is_filtered = True virtual_codebase.save_resource(virtual_codebase.root) @@ -864,7 +885,8 @@ def test_virtual_codebase_walk_filtered_with_filtered_root(self): assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered_with_all_filtered(self): - scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/virtual_codebase.json') virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): res.is_filtered = True @@ -875,14 +897,16 @@ def test_virtual_codebase_walk_filtered_with_all_filtered(self): assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_compute_counts_filtered_None(self): - scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/virtual_codebase.json') virtual_codebase = VirtualCodebase(location=scan_data) results = virtual_codebase.compute_counts(skip_filtered=True) expected = (5, 3, 2228) assert results == expected def test_virtual_codebase_compute_counts_filtered_None_with_size(self): - scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/virtual_codebase.json') virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): if res.is_file: @@ -894,14 +918,16 @@ def test_virtual_codebase_compute_counts_filtered_None_with_size(self): assert results == expected def test_virtual_codebase_compute_counts_filtered_None_with_cache(self): - scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/virtual_codebase.json') virtual_codebase = VirtualCodebase(location=scan_data) results = virtual_codebase.compute_counts(skip_filtered=True) expected = (5, 3, 2228) assert results == expected def test_virtual_codebase_compute_counts_filtered_all(self): - scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/virtual_codebase.json') virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): res.is_filtered = True @@ -911,7 +937,8 @@ def test_virtual_codebase_compute_counts_filtered_all(self): assert results == expected def test_virtual_codebase_compute_counts_filtered_all_with_cache(self): - scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/virtual_codebase.json') virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): res.is_filtered = True @@ -921,7 +948,8 @@ def test_virtual_codebase_compute_counts_filtered_all_with_cache(self): assert results == expected def test_virtual_codebase_compute_counts_filtered_files(self): - scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/virtual_codebase.json') virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): if res.is_file: @@ -932,7 +960,8 @@ def test_virtual_codebase_compute_counts_filtered_files(self): assert results == expected def test_virtual_codebase_compute_counts_filtered_dirs(self): - scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/virtual_codebase.json') virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): if not res.is_file: @@ -943,7 +972,8 @@ def test_virtual_codebase_compute_counts_filtered_dirs(self): assert results == expected def test_virtual_codebase_walk_filtered_dirs(self): - scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/virtual_codebase.json') virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): if not res.is_file: @@ -960,7 +990,8 @@ def test_virtual_codebase_walk_filtered_dirs(self): assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered_skip_root(self): - scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/virtual_codebase.json') virtual_codebase = VirtualCodebase(location=scan_data) virtual_codebase.root.is_filtered = True virtual_codebase.save_resource(virtual_codebase.root) @@ -977,7 +1008,8 @@ def test_virtual_codebase_walk_filtered_skip_root(self): assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered_all_skip_root(self): - scan_data = self.get_test_loc('resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/virtual_codebase.json') virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): res.is_filtered = True @@ -988,21 +1020,24 @@ def test_virtual_codebase_walk_filtered_all_skip_root(self): assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_skip_root_single_file(self): - scan_data = self.get_test_loc('resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/et131x.h.json') virtual_codebase = VirtualCodebase(location=scan_data) results = list(virtual_codebase.walk(skip_root=True)) expected = [('et131x.h', True)] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered_with_skip_root_and_single_file_not_filtered(self): - scan_data = self.get_test_loc('resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/et131x.h.json') virtual_codebase = VirtualCodebase(location=scan_data) results = list(virtual_codebase.walk_filtered(skip_root=True)) expected = [('et131x.h', True)] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered__with_skip_root_and_filtered_single_file(self): - scan_data = self.get_test_loc('resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/et131x.h.json') virtual_codebase = VirtualCodebase(location=scan_data) virtual_codebase.root.is_filtered = True virtual_codebase.save_resource(virtual_codebase.root) @@ -1011,7 +1046,8 @@ def test_virtual_codebase_walk_filtered__with_skip_root_and_filtered_single_file assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_skip_root_single_file_with_children(self): - scan_data = self.get_test_loc('resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/et131x.h.json') virtual_codebase = VirtualCodebase(location=scan_data) c1 = virtual_codebase._get_or_create_resource( @@ -1025,11 +1061,13 @@ def test_virtual_codebase_walk_skip_root_single_file_with_children(self): is_file=False, ) results = list(virtual_codebase.walk(skip_root=True)) - expected = [('et131x.h', True), ('some child', True), ('some child2', False)] + expected = [('et131x.h', True), ('some child', True), + ('some child2', False)] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered_with_skip_root_and_single_file_with_children(self): - scan_data = self.get_test_loc('resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/et131x.h.json') virtual_codebase = VirtualCodebase(location=scan_data) c1 = virtual_codebase._get_or_create_resource( @@ -1058,7 +1096,8 @@ def test_virtual_codebase_walk_filtered_with_skip_root_and_single_file_with_chil assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase__create_resource_can_add_child_to_file(self): - scan_data = self.get_test_loc('resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/et131x.h.json') virtual_codebase = VirtualCodebase(location=scan_data) virtual_codebase._get_or_create_resource( 'some child', @@ -1070,7 +1109,8 @@ def test_virtual_codebase__create_resource_can_add_child_to_file(self): assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase__create_resource_can_add_child_to_dir(self): - scan_data = self.get_test_loc('resource/virtual_codebase/resource.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/resource.json') virtual_codebase = VirtualCodebase(location=scan_data) virtual_codebase._get_or_create_resource( 'some child', @@ -1082,10 +1122,13 @@ def test_virtual_codebase__create_resource_can_add_child_to_dir(self): assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_get_resource(self): - scan_data = self.get_test_loc('resource/virtual_codebase/resource.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/resource.json') virtual_codebase = VirtualCodebase(location=scan_data) - assert not (virtual_codebase.root is virtual_codebase.get_resource('resource')) - assert virtual_codebase.get_resource('resource') == virtual_codebase.root + assert not ( + virtual_codebase.root is virtual_codebase.get_resource('resource')) + assert virtual_codebase.get_resource( + 'resource') == virtual_codebase.root def test_virtual_codebase_can_process_minimal_resources_without_info(self): scan_data = self.get_test_loc('resource/virtual_codebase/noinfo.json') @@ -1109,7 +1152,8 @@ def test_virtual_codebase_can_process_minimal_resources_without_info(self): assert [r.to_dict() for r in codebase.walk()] == expected def test_virtual_codebase_can_process_minimal_resources_with_only_path(self): - scan_data = self.get_test_loc('resource/virtual_codebase/only-path.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/only-path.json') codebase = VirtualCodebase(location=scan_data) expected = [ {'path': 'samples', 'type': 'directory', 'scan_errors': []}, @@ -1118,16 +1162,20 @@ def test_virtual_codebase_can_process_minimal_resources_with_only_path(self): assert [r.to_dict() for r in codebase.walk()] == expected def test_VirtualCodebase_account_fingerprint_attribute(self): - test_file = self.get_test_loc("resource/virtual_codebase/fingerprint_attribute.json") + test_file = self.get_test_loc( + "resource/virtual_codebase/fingerprint_attribute.json") codebase = VirtualCodebase(test_file) - resources_fingerprint = [resource.fingerprint for resource in codebase.walk()] + resources_fingerprint = [ + resource.fingerprint for resource in codebase.walk()] assert "e30cf09443e7878dfed3288886e97542" in resources_fingerprint assert None in resources_fingerprint - assert codebase.get_resource('apache_to_all_notable_lic_new') == codebase.root + assert codebase.get_resource( + 'apache_to_all_notable_lic_new') == codebase.root assert resources_fingerprint.count(None) == 2 def test_VirtualCodebase_works_with_mapping_backed_codebase(self): - test_file = self.get_test_loc("resource/virtual_codebase/license-scan.json") + test_file = self.get_test_loc( + "resource/virtual_codebase/license-scan.json") codebase = VirtualCodebase(test_file) resource = codebase.get_resource('scan-ref/license-notice.txt') assert resource @@ -1145,7 +1193,8 @@ def test_lowest_common_parent_on_virtual_codebase(self): assert lcp.name == 'test1' def test_virtual_codebase_has_default_for_plugin_attributes(self): - scan_data = self.get_test_loc('resource/virtual_codebase/only-path.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/only-path.json') VirtualCodebase(location=scan_data) def test_lowest_common_parent_strip(self): @@ -1200,28 +1249,33 @@ class TestVirtualCodebaseCache(FileBasedTesting): test_data_dir = join(dirname(__file__), 'data') def test_virtual_codebase_cache_default(self): - scan_data = self.get_test_loc('resource/virtual_codebase/codebase-for-cache-tests.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/codebase-for-cache-tests.json') virtual_codebase = VirtualCodebase(location=scan_data) assert virtual_codebase.temp_dir assert virtual_codebase.cache_dir virtual_codebase.cache_dir root = virtual_codebase.root - cp = virtual_codebase._get_resource_cache_location(root.path, create_dirs=False) + cp = virtual_codebase._get_resource_cache_location( + root.path, create_dirs=False) assert not exists(cp) - cp = virtual_codebase._get_resource_cache_location(root.path, create_dirs=True) + cp = virtual_codebase._get_resource_cache_location( + root.path, create_dirs=True) assert not exists(cp) assert exists(parent_directory(cp)) - child = virtual_codebase._get_or_create_resource('child', root, is_file=True) + child = virtual_codebase._get_or_create_resource( + 'child', root, is_file=True) child.size = 12 virtual_codebase.save_resource(child) child_2 = virtual_codebase.get_resource(child.path) assert child_2 == child def test_virtual_codebase_cache_all_in_memory(self): - scan_data = self.get_test_loc('resource/virtual_codebase/codebase-for-cache-tests.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/codebase-for-cache-tests.json') virtual_codebase = VirtualCodebase(location=scan_data, max_in_memory=0) for path, res in virtual_codebase.resources_by_path.items(): assert res != Codebase.CACHED_RESOURCE @@ -1242,8 +1296,10 @@ def test_virtual_codebase_cache_all_in_memory(self): ) def test_virtual_codebase_cache_all_on_disk(self): - scan_data = self.get_test_loc('resource/virtual_codebase/codebase-for-cache-tests.json') - virtual_codebase = VirtualCodebase(location=scan_data, max_in_memory=-1) + scan_data = self.get_test_loc( + 'resource/virtual_codebase/codebase-for-cache-tests.json') + virtual_codebase = VirtualCodebase( + location=scan_data, max_in_memory=-1) for path, res in virtual_codebase.resources_by_path.items(): if res != Codebase.CACHED_RESOURCE: @@ -1252,7 +1308,8 @@ def test_virtual_codebase_cache_all_on_disk(self): res = virtual_codebase.get_resource(path) if res.is_root: - assert virtual_codebase.get_resource(path) == virtual_codebase.root + assert virtual_codebase.get_resource( + path) == virtual_codebase.root assert virtual_codebase._exists_in_memory(path) assert not virtual_codebase._exists_on_disk(path) else: @@ -1266,7 +1323,8 @@ def test_virtual_codebase_cache_all_on_disk(self): ) def test_virtual_codebase_cache_mixed_two_in_memory(self): - scan_data = self.get_test_loc('resource/virtual_codebase/codebase-for-cache-tests.json') + scan_data = self.get_test_loc( + 'resource/virtual_codebase/codebase-for-cache-tests.json') virtual_codebase = VirtualCodebase(location=scan_data, max_in_memory=2) counter = 0 @@ -1276,7 +1334,8 @@ def test_virtual_codebase_cache_mixed_two_in_memory(self): if res.is_root: assert ( - virtual_codebase.get_resource(path).to_dict() == virtual_codebase.root.to_dict() + virtual_codebase.get_resource( + path).to_dict() == virtual_codebase.root.to_dict() ) assert virtual_codebase._exists_in_memory(path) assert not virtual_codebase._exists_on_disk(path) @@ -1302,7 +1361,8 @@ class TestVirtualCodebaseCreation(FileBasedTesting): test_data_dir = join(dirname(__file__), 'data') def test_VirtualCodebase_can_be_created_from_json_file(self): - test_file = self.get_test_loc('resource/virtual_codebase/from_file.json') + test_file = self.get_test_loc( + 'resource/virtual_codebase/from_file.json') codebase = VirtualCodebase(test_file) results = sorted(r.name for r in codebase.walk()) expected = ['bar.svg', 'han'] @@ -1357,7 +1417,8 @@ def test_VirtualCodebase_can_be_created_from_dict(self): assert results == expected def test_VirtualCodebase_create_from_scan_with_no_root_and_missing_parents(self): - test_file = self.get_test_loc('resource/virtual_codebase/samples-only-findings.json') + test_file = self.get_test_loc( + 'resource/virtual_codebase/samples-only-findings.json') result_file = self.get_test_loc( 'resource/virtual_codebase/samples-only-findings-expected.json' ) @@ -1368,18 +1429,24 @@ def test_VirtualCodebase_create_from_scan_with_no_root_and_missing_parents(self) assert results == expected def test_VirtualCodebase_check_that_already_existing_parent_is_updated_properly(self): - test_file = self.get_test_loc('resource/virtual_codebase/root-is-not-first-resource.json') + test_file = self.get_test_loc( + 'resource/virtual_codebase/root-is-not-first-resource.json') codebase = VirtualCodebase(test_file) - results = sorted((r.to_dict() for r in codebase.walk()), key=lambda x: tuple(x.items())) + results = sorted((r.to_dict() for r in codebase.walk()), + key=lambda x: tuple(x.items())) expected = [ - {'path': 'samples', 'type': 'directory', 'summary': ['asd'], 'scan_errors': []}, - {'path': 'samples/NOTICE', 'type': 'file', 'summary': [], 'scan_errors': []}, + {'path': 'samples', 'type': 'directory', + 'summary': ['asd'], 'scan_errors': []}, + {'path': 'samples/NOTICE', 'type': 'file', + 'summary': [], 'scan_errors': []}, ] assert results == expected def test_VirtualCodebase_create_from_multiple_scans(self): - test_file_1 = self.get_test_loc('resource/virtual_codebase/combine-1.json') - test_file_2 = self.get_test_loc('resource/virtual_codebase/combine-2.json') + test_file_1 = self.get_test_loc( + 'resource/virtual_codebase/combine-1.json') + test_file_2 = self.get_test_loc( + 'resource/virtual_codebase/combine-2.json') vinput = (test_file_1, test_file_2) codebase = VirtualCodebase(vinput) results = [r.to_dict(with_info=False) for r in codebase.walk()] @@ -1407,7 +1474,8 @@ def test_VirtualCodebase_create_from_multiple_scans_shared_directory_names(self) check_against_expected_json_file(results, expected_file, regen=False) def test_VirtualCodebase_compute_counts_with_full_root_info_one(self): - test_file = self.get_test_loc('resource/virtual_codebase/full-root-info-one.json') + test_file = self.get_test_loc( + 'resource/virtual_codebase/full-root-info-one.json') codebase = VirtualCodebase(test_file) resource = [r for r in codebase.walk() if r.is_file][0] assert resource.path == 'home/foobar/scancode-toolkit/samples/README' @@ -1417,7 +1485,8 @@ def test_VirtualCodebase_compute_counts_with_full_root_info_one(self): assert size_count == 236 def test_VirtualCodebase_with_full_root_info_one(self): - test_file = self.get_test_loc('resource/virtual_codebase/full-root-info-one.json') + test_file = self.get_test_loc( + 'resource/virtual_codebase/full-root-info-one.json') codebase = VirtualCodebase(test_file) results = [r.to_dict(with_info=True) for r in codebase.walk()] expected_file = self.get_test_loc( @@ -1426,7 +1495,8 @@ def test_VirtualCodebase_with_full_root_info_one(self): check_against_expected_json_file(results, expected_file, regen=False) def test_VirtualCodebase_with_full_root_info_many(self): - test_file = self.get_test_loc('resource/virtual_codebase/full-root-info-many.json') + test_file = self.get_test_loc( + 'resource/virtual_codebase/full-root-info-many.json') codebase = VirtualCodebase(test_file) results = [r.to_dict(with_info=True) for r in codebase.walk()] expected_file = self.get_test_loc( @@ -1438,17 +1508,20 @@ def test_VirtualCodebase_can_compute_counts_with_null(self): # was failing with # size_count += child.size # TypeError: unsupported operand type(s) for +=: 'int' and 'NoneType' - test_file = self.get_test_loc('resource/virtual_codebase/node-16-slim.json') + test_file = self.get_test_loc( + 'resource/virtual_codebase/node-16-slim.json') codebase = VirtualCodebase(test_file) codebase.compute_counts() def test_VirtualCodebase_can_be_created_with_single_path(self): - test_file = self.get_test_loc('resource/virtual_codebase/docker-hello-world.json') + test_file = self.get_test_loc( + 'resource/virtual_codebase/docker-hello-world.json') VirtualCodebase(test_file) def test_VirtualCodebase_can_be_created_without_RecursionError(self): # was failing with RecursionError: maximum recursion depth exceeded - test_file = self.get_test_loc('resource/virtual_codebase/zephyr-binary.json') + test_file = self.get_test_loc( + 'resource/virtual_codebase/zephyr-binary.json') VirtualCodebase(test_file) def test_VirtualCodebase_can_be_created_with_repeated_root_directory(self): @@ -1468,7 +1541,8 @@ class TestResource(FileBasedTesting): test_data_dir = join(dirname(__file__), 'data') def test_Resource_extracted_to_extracted_from(self): - test_file = self.get_test_loc('resource/resource/test-extracted-from-to.json') + test_file = self.get_test_loc( + 'resource/resource/test-extracted-from-to.json') codebase = VirtualCodebase(location=test_file) results = [] for r in codebase.walk(topdown=True): @@ -1534,7 +1608,8 @@ def test_Resource_extracted_to_extracted_from(self): assert results == expected def test_virtualcode_Resource_can_walk(self): - test_file = self.get_test_loc('resource/resource/test-extracted-from-to.json') + test_file = self.get_test_loc( + 'resource/resource/test-extracted-from-to.json') codebase = VirtualCodebase(location=test_file) results = [r.path for r in codebase.walk(topdown=True)] diff --git a/tests/test_saneyaml.py b/tests/test_saneyaml.py index 984cdf11..2010bff2 100644 --- a/tests/test_saneyaml.py +++ b/tests/test_saneyaml.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/tests/test_text.py b/tests/test_text.py index dcb1beb0..4b08bc29 100644 --- a/tests/test_text.py +++ b/tests/test_text.py @@ -3,7 +3,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -24,7 +24,8 @@ def test_lines(): ''' assert len([p[1] for p in text.lines(t)]) == 5 - expected = ['This problem is.', 'It is therefore', 'However,we', 'without introducing ..', 'However, I have'] + expected = ['This problem is.', 'It is therefore', + 'However,we', 'without introducing ..', 'However, I have'] assert [p for p in text.lines(t)] == expected @@ -35,7 +36,8 @@ def test_foldcase(): def test_nopunctuation(): test = '''This problem is about sequence-bunching, %^$^%**^&*©©^(*&(*()()_+)_!@@#:><>>?/./,.,';][{}{]just''' - expected = ['This', 'problem', 'is', 'about', 'sequence', 'bunching', 'Â', 'Â', 'just'] + expected = ['This', 'problem', 'is', 'about', + 'sequence', 'bunching', 'Â', 'Â', 'just'] assert text.nopunctuation(test).split() == expected test = 'This problem is about: sequence-bunching\n\n just \n' diff --git a/tests/test_timeutils.py b/tests/test_timeutils.py index 7ed973fa..1cd58f2f 100644 --- a/tests/test_timeutils.py +++ b/tests/test_timeutils.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -48,39 +48,48 @@ def test_tstamp2time_format(self): def test_tstamp2time(self): dt_from_ts = tstamp2time('2010-11-12T131415.000016') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, second=15, microsecond=16, tzinfo=UTC()) == dt_from_ts + assert datetime(year=2010, month=11, day=12, hour=13, minute=14, + second=15, microsecond=16, tzinfo=UTC()) == dt_from_ts def test_tstamp2time2(self): dt_from_ts = tstamp2time('20101112T131415.000016') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, second=15, microsecond=16, tzinfo=UTC()) == dt_from_ts + assert datetime(year=2010, month=11, day=12, hour=13, minute=14, + second=15, microsecond=16, tzinfo=UTC()) == dt_from_ts def test_tstamp2time3(self): dt_from_ts = tstamp2time('20101112T131415.000016Z') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, second=15, microsecond=16, tzinfo=UTC()) == dt_from_ts + assert datetime(year=2010, month=11, day=12, hour=13, minute=14, + second=15, microsecond=16, tzinfo=UTC()) == dt_from_ts def test_tstamp2time4(self): dt_from_ts = tstamp2time('2010-11-12T131415') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts + assert datetime(year=2010, month=11, day=12, hour=13, minute=14, + second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts def test_tstamp2time5(self): dt_from_ts = tstamp2time('2010-11-12T13:14:15') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts + assert datetime(year=2010, month=11, day=12, hour=13, minute=14, + second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts def test_tstamp2time6(self): dt_from_ts = tstamp2time('20101112T13:14:15') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts + assert datetime(year=2010, month=11, day=12, hour=13, minute=14, + second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts def test_tstamp2time7(self): dt_from_ts = tstamp2time('20101112T13:14:15Z') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts + assert datetime(year=2010, month=11, day=12, hour=13, minute=14, + second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts def test_tstamp2time8(self): dt_from_ts = tstamp2time('20101112T13:14:15Z') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts + assert datetime(year=2010, month=11, day=12, hour=13, minute=14, + second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts def test_tstamp2time9(self): dt_from_ts = tstamp2time('2010-06-30T21:26:40.000Z') - assert datetime(year=2010, month=6, day=30, hour=21, minute=26, second=40, microsecond=0, tzinfo=UTC()) == dt_from_ts + assert datetime(year=2010, month=6, day=30, hour=21, minute=26, + second=40, microsecond=0, tzinfo=UTC()) == dt_from_ts def test_tstamp2time_raise(self): self.assertRaises(ValueError, tstamp2time, '201011A12T13:14:15Z') diff --git a/tests/test_urn.py b/tests/test_urn.py index f8443fd1..1a85c786 100644 --- a/tests/test_urn.py +++ b/tests/test_urn.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -89,7 +89,7 @@ def test_decode_org(self): def test_decode_build_is_idempotent(self): u1 = urn.encode('component', owner__name='org%', name='name%', - version='version%') + version='version%') m, f = urn.decode(u1) u3 = urn.encode(m, **f) assert u3 == u1 @@ -100,11 +100,11 @@ def test_decode_raise_exception_if_incorrect_prefix(self): def test_decode_raise_exception_if_incorrect_ns(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:x:x:x') + urn.decode('urn:x:x:x') def test_decode_raise_exception_if_incorrect_prefix_or_ns(self): with self.assertRaises(urn.URNValidationError): - urn.decode('x:x:x:x') + urn.decode('x:x:x:x') def test_decode_raise_exception_if_too_short_license(self): with self.assertRaises(urn.URNValidationError): diff --git a/tests/test_version.py b/tests/test_version.py index 00fc8b42..9f54b841 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/commoncode for support or download. +# See https://github.com/aboutcode-org/commoncode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -113,7 +113,7 @@ def test_version_hint(self): '/cassandra/debian/pool/main/c/cassandra/cassandra_0.4.0~beta1-1.diff.gz': '0.4.0~beta1-1', '/ha-api-3.1.6.jar': '3.1.6', 'ha-api-3.1.6.jar': '3.1.6' - } + } # FIXME: generate a test function for each case for path in data: From d1af11e0165afdac3063fe404a03e480045f15bf Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Wed, 14 Aug 2024 12:35:33 +0800 Subject: [PATCH 013/120] Remove the deprecated macos_11 Signed-off-by: Chin Yeung Li --- azure-pipelines.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a5aab8e6..59ab431b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -97,13 +97,3 @@ jobs: all: venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e . && venv\Scripts\pytest -n 2 -vvs - - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos11_cpython_latest_from_pip - image_name: macos-11 - python_versions: ["3.7", "3.8", "3.9", "3.10"] - test_suites: - all: - venv/bin/pip install --upgrade-strategy eager --force-reinstall - --upgrade -e . && venv/bin/pytest -n 2 -vvs From 9c57f340d22d8891a5614a93553b20d75e2f3136 Mon Sep 17 00:00:00 2001 From: Chin Yeung Li Date: Tue, 20 Aug 2024 16:46:20 +0800 Subject: [PATCH 014/120] Update link references of ownership from nexB to aboutcode-org Signed-off-by: Chin Yeung Li --- Makefile | 4 +- NOTICE | 2 +- configure | 2 +- configure.bat | 2 +- docs/source/conf.py | 2 +- docs/source/contribute/contrib_doc.rst | 2 +- docs/source/skeleton-usage.rst | 2 +- etc/scripts/check_thirdparty.py | 5 +- etc/scripts/fetch_thirdparty.py | 19 ++++-- etc/scripts/gen_requirements.py | 2 +- etc/scripts/gen_requirements_dev.py | 2 +- etc/scripts/utils_dejacode.py | 11 ++-- etc/scripts/utils_requirements.py | 11 ++-- etc/scripts/utils_thirdparty.py | 89 +++++++++++++++++--------- setup.cfg | 2 +- tests/test_skeleton_codestyle.py | 2 +- 16 files changed, 100 insertions(+), 59 deletions(-) diff --git a/Makefile b/Makefile index cc36c355..94451b33 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -35,7 +35,7 @@ check: @echo "-> Run pycodestyle (PEP8) validation" @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,migrations,settings.py,.cache . @echo "-> Run isort imports ordering validation" - @${ACTIVATE} isort --sl --check-only -l 100 setup.py src tests . + @${ACTIVATE} isort --sl --check-only -l 100 setup.py src tests . @echo "-> Run black validation" @${ACTIVATE} black --check --check -l 100 src tests setup.py diff --git a/NOTICE b/NOTICE index 65936b2b..cbdaef79 100644 --- a/NOTICE +++ b/NOTICE @@ -2,7 +2,7 @@ # Copyright (c) nexB Inc. and others. # SPDX-License-Identifier: Apache-2.0 # -# Visit https://aboutcode.org and https://github.com/nexB/ for support and download. +# Visit https://aboutcode.org and https://github.com/aboutcode-org/ for support and download. # ScanCode is a trademark of nexB Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/configure b/configure index 926a894e..22d92885 100755 --- a/configure +++ b/configure @@ -3,7 +3,7 @@ # Copyright (c) nexB Inc. and others. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/ for support or download. +# See https://github.com/aboutcode-org/ for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # diff --git a/configure.bat b/configure.bat index 5e95b311..5b9a9d68 100644 --- a/configure.bat +++ b/configure.bat @@ -4,7 +4,7 @@ @rem Copyright (c) nexB Inc. and others. All rights reserved. @rem SPDX-License-Identifier: Apache-2.0 @rem See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -@rem See https://github.com/nexB/ for support or download. +@rem See https://github.com/aboutcode-org/ for support or download. @rem See https://aboutcode.org for more information about nexB OSS projects. diff --git a/docs/source/conf.py b/docs/source/conf.py index 7771ff09..8c88fa2c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -43,7 +43,7 @@ # This points to aboutcode.readthedocs.io # In case of "undefined label" ERRORS check docs on intersphinx to troubleshoot -# Link was created at commit - https://github.com/nexB/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83 +# Link was created at commit - https://github.com/aboutcode-org/aboutcode/commit/faea9fcf3248f8f198844fe34d43833224ac4a83 intersphinx_mapping = { "aboutcode": ("https://aboutcode.readthedocs.io/en/latest/", None), diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst index 13882e10..5640db26 100644 --- a/docs/source/contribute/contrib_doc.rst +++ b/docs/source/contribute/contrib_doc.rst @@ -12,7 +12,7 @@ To get started, create or identify a working directory on your local machine. Open that directory and execute the following command in a terminal session:: - git clone https://github.com/nexB/skeleton.git + git clone https://github.com/aboutcode-org/skeleton.git That will create an ``/skeleton`` directory in your working directory. Now you can install the dependencies in a virtualenv:: diff --git a/docs/source/skeleton-usage.rst b/docs/source/skeleton-usage.rst index cde23dcd..6cb4cc5f 100644 --- a/docs/source/skeleton-usage.rst +++ b/docs/source/skeleton-usage.rst @@ -118,7 +118,7 @@ corrected. You can check to see if your corrections are valid by running: Once the wheels are collected and the ABOUT files are generated and correct, upload them to thirdparty.aboutcode.org/pypi by placing the wheels and ABOUT files from the thirdparty directory to the pypi directory at -https://github.com/nexB/thirdparty-packages +https://github.com/aboutcode-org/thirdparty-packages Usage after project initialization diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index b052f25b..2daded94 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import click @@ -17,7 +17,8 @@ @click.option( "-d", "--dest", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + type=click.Path(exists=True, readable=True, + path_type=str, file_okay=False), required=True, help="Path to the thirdparty directory to check.", ) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index eedf05c6..3f9ff527 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -55,7 +55,8 @@ "-d", "--dest", "dest_dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + type=click.Path(exists=True, readable=True, + path_type=str, file_okay=False), metavar="DIR", default=utils_thirdparty.THIRDPARTY_DIR, show_default=True, @@ -224,7 +225,8 @@ def fetch_thirdparty( environments = None if wheels: evts = itertools.product(python_versions, operating_systems) - environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] + environments = [utils_thirdparty.Environment.from_pyver_and_os( + pyv, os) for pyv, os in evts] # Collect PyPI repos repos = [] @@ -260,13 +262,14 @@ def fetch_thirdparty( repos=repos, ) if not fetched: - wheels_or_sdist_not_found[f"{name}=={version}"].append(environment) + wheels_or_sdist_not_found[f"{name}=={version}"].append( + environment) if TRACE: print(f" NOT FOUND") if (sdists or (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only) - ): + ): if TRACE: print(f" ==> Fetching sdist: {name}=={version}") @@ -289,7 +292,8 @@ def fetch_thirdparty( sdist_missing = sdists and "sdist" in dists and not name in wheel_only if sdist_missing: mia.append(f"SDist missing: {nv} {dists}") - wheels_missing = wheels and any(d for d in dists if d != "sdist") and not name in sdist_only + wheels_missing = wheels and any( + d for d in dists if d != "sdist") and not name in sdist_only if wheels_missing: mia.append(f"Wheels missing: {nv} {dists}") @@ -299,7 +303,8 @@ def fetch_thirdparty( raise Exception(mia) print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") - utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) + utils_thirdparty.fetch_abouts_and_licenses( + dest_dir=dest_dir, use_cached_index=use_cached_index) utils_thirdparty.clean_about_files(dest_dir=dest_dir) # check for problems diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py index 07e26f77..2b65ae80 100644 --- a/etc/scripts/gen_requirements.py +++ b/etc/scripts/gen_requirements.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import argparse diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py index 12cc06d3..5db1c48e 100644 --- a/etc/scripts/gen_requirements_dev.py +++ b/etc/scripts/gen_requirements_dev.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import argparse diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index c42e6c93..652252d4 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import io @@ -33,7 +33,8 @@ def can_do_api_calls(): if not DEJACODE_API_KEY and DEJACODE_API_URL: - print("DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") + print( + "DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") return False else: return True @@ -68,7 +69,8 @@ def get_package_data(distribution): return results[0] elif len_results > 1: - print(f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") + print( + f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") else: print("Could not find package:", distribution.download_url) @@ -149,7 +151,8 @@ def find_latest_dejacode_package(distribution): # there was no exact match, find the latest version # TODO: consider the closest version rather than the latest # or the version that has the best data - with_versions = [(packaging_version.parse(p["version"]), p) for p in packages] + with_versions = [(packaging_version.parse(p["version"]), p) + for p in packages] with_versions = sorted(with_versions) latest_version, latest_package_version = sorted(with_versions)[-1] print( diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 0fc25a35..1c502390 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -102,7 +102,8 @@ def lock_dev_requirements( all_req_nvs = get_required_name_versions(all_req_lines) dev_only_req_nvs = {n: v for n, v in all_req_nvs if n not in main_names} - new_reqs = "\n".join(f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) + new_reqs = "\n".join( + f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) with open(dev_requirements_file, "w") as fo: fo.write(new_reqs) @@ -113,10 +114,12 @@ def get_installed_reqs(site_packages_dir): as a text. """ if not os.path.exists(site_packages_dir): - raise Exception(f"site_packages directory: {site_packages_dir!r} does not exists") + raise Exception( + f"site_packages directory: {site_packages_dir!r} does not exists") # Also include these packages in the output with --all: wheel, distribute, # setuptools, pip - args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] + args = ["pip", "freeze", "--exclude-editable", + "--all", "--path", site_packages_dir] return subprocess.check_output(args, encoding="utf-8") diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index addf8e5e..46dc7289 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -5,7 +5,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import email @@ -245,9 +245,11 @@ def download_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR, repos=tu package = repo.get_package_version(name=name, version=version) if not package: if TRACE_DEEP: - print(f" download_wheel: No package in {repo.index_url} for {name}=={version}") + print( + f" download_wheel: No package in {repo.index_url} for {name}=={version}") continue - supported_wheels = list(package.get_supported_wheels(environment=environment)) + supported_wheels = list( + package.get_supported_wheels(environment=environment)) if not supported_wheels: if TRACE_DEEP: print( @@ -291,7 +293,8 @@ def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): if not package: if TRACE_DEEP: - print(f" download_sdist: No package in {repo.index_url} for {name}=={version}") + print( + f" download_sdist: No package in {repo.index_url} for {name}=={version}") continue sdist = package.sdist if not sdist: @@ -300,7 +303,8 @@ def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): continue if TRACE_DEEP: - print(f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") + print( + f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") fetched_sdist_filename = package.sdist.download(dest_dir=dest_dir) if fetched_sdist_filename: @@ -533,7 +537,8 @@ def get_best_download_url(self, repos=tuple()): repos = DEFAULT_PYPI_REPOS for repo in repos: - package = repo.get_package_version(name=self.name, version=self.version) + package = repo.get_package_version( + name=self.name, version=self.version) if not package: if TRACE: print( @@ -772,7 +777,8 @@ def load_remote_about_data(self): if notice_text: about_data["notice_text"] = notice_text except RemoteNotFetchedException: - print(f"Failed to fetch NOTICE file: {self.notice_download_url}") + print( + f"Failed to fetch NOTICE file: {self.notice_download_url}") return self.load_about_data(about_data) def get_checksums(self, dest_dir=THIRDPARTY_DIR): @@ -821,9 +827,11 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): Fetch license files if missing in `dest_dir`. Return True if license files were fetched. """ - urls = LinksRepository.from_url(use_cached_index=use_cached_index).links + urls = LinksRepository.from_url( + use_cached_index=use_cached_index).links errors = [] - extra_lic_names = [l.get("file") for l in self.extra_data.get("licenses", {})] + extra_lic_names = [l.get("file") + for l in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] extra_lic_names = [ln for ln in extra_lic_names if ln] lic_names = [f"{key}.LICENSE" for key in self.get_license_keys()] @@ -834,7 +842,8 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): try: # try remotely first - lic_url = get_license_link_for_filename(filename=filename, urls=urls) + lic_url = get_license_link_for_filename( + filename=filename, urls=urls) fetch_and_save( path_or_url=lic_url, @@ -911,7 +920,8 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): c for c in classifiers if c.startswith("License") ] license_expression = get_license_expression(declared_license) - other_classifiers = [c for c in classifiers if not c.startswith("License")] + other_classifiers = [ + c for c in classifiers if not c.startswith("License")] holder = raw_data["Author"] holder_contact = raw_data["Author-email"] @@ -953,7 +963,8 @@ def update(self, data, overwrite=False, keep_extra=True): package_url = data.get("package_url") if package_url: purl_from_data = packageurl.PackageURL.from_string(package_url) - purl_from_self = packageurl.PackageURL.from_string(self.package_url) + purl_from_self = packageurl.PackageURL.from_string( + self.package_url) if purl_from_data != purl_from_self: print( f"Invalid dist update attempt, no same same purl with dist: " @@ -1003,7 +1014,8 @@ def get_license_link_for_filename(filename, urls): if not path_or_url: raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: - raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) + raise Exception( + f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) return path_or_url[0] @@ -1397,7 +1409,8 @@ def packages_from_dir(cls, directory): """ base = os.path.abspath(directory) - paths = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] + paths = [os.path.join(base, f) + for f in os.listdir(base) if f.endswith(EXTENSIONS)] if TRACE_ULTRA_DEEP: print("packages_from_dir: paths:", paths) @@ -1458,7 +1471,8 @@ def dists_from_paths_or_urls(cls, paths_or_urls): dists = [] if TRACE_ULTRA_DEEP: print(" ###paths_or_urls:", paths_or_urls) - installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)] + installable = [f for f in paths_or_urls if f.endswith( + EXTENSIONS_INSTALLABLE)] for path_or_url in installable: try: dist = Distribution.from_path_or_url(path_or_url) @@ -1476,7 +1490,8 @@ def dists_from_paths_or_urls(cls, paths_or_urls): ) except InvalidDistributionFilename: if TRACE_DEEP: - print(f" Skipping invalid distribution from: {path_or_url}") + print( + f" Skipping invalid distribution from: {path_or_url}") continue return dists @@ -1525,7 +1540,8 @@ class Environment: implementation = attr.ib( type=str, default="cp", - metadata=dict(help="Python implementation supported by this environment."), + metadata=dict( + help="Python implementation supported by this environment."), repr=False, ) @@ -1539,7 +1555,8 @@ class Environment: platforms = attr.ib( type=list, default=attr.Factory(list), - metadata=dict(help="List of platform tags supported by this environment."), + metadata=dict( + help="List of platform tags supported by this environment."), repr=False, ) @@ -1623,7 +1640,8 @@ class PypiSimpleRepository: fetched_package_normalized_names = attr.ib( type=set, default=attr.Factory(set), - metadata=dict(help="A set of already fetched package normalized names."), + metadata=dict( + help="A set of already fetched package normalized names."), ) use_cached_index = attr.ib( @@ -1654,10 +1672,12 @@ def _get_package_versions_map(self, name): self.packages[normalized_name] = versions except RemoteNotFetchedException as e: if TRACE: - print(f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") + print( + f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") if not versions and TRACE: - print(f"WARNING: package {name} not found in repo: {self.index_url}") + print( + f"WARNING: package {name} not found in repo: {self.index_url}") return versions @@ -1842,7 +1862,8 @@ def get(self, path_or_url, as_text=True, force=False): if force or not os.path.exists(cached): if TRACE_DEEP: print(f" FILE CACHE MISS: {path_or_url}") - content = get_file_content(path_or_url=path_or_url, as_text=as_text) + content = get_file_content( + path_or_url=path_or_url, as_text=as_text) wmode = "w" if as_text else "wb" with open(cached, wmode) as fo: fo.write(content) @@ -1864,7 +1885,8 @@ def get_file_content(path_or_url, as_text=True): if path_or_url.startswith("https://"): if TRACE_DEEP: print(f"Fetching: {path_or_url}") - _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) + _headers, content = get_remote_file_content( + url=path_or_url, as_text=as_text) return content elif path_or_url.startswith("file://") or ( @@ -1930,7 +1952,8 @@ def get_remote_file_content( ) else: - raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") + raise RemoteNotFetchedException( + f"Failed HTTP request from {url} with {status}") if headers_only: return response.headers, None @@ -2021,7 +2044,8 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files( + dest_dir=dest_dir, use_cached_index=use_cached_index) continue # lets try to get from another dist of the same local package @@ -2033,7 +2057,8 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files( + dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get another version of the same package that is not our version @@ -2044,7 +2069,8 @@ def get_other_dists(_package, _dist): ] other_local_version = other_local_packages and other_local_packages[-1] if other_local_version: - latest_local_dists = list(other_local_version.get_distributions()) + latest_local_dists = list( + other_local_version.get_distributions()) for latest_local_dist in latest_local_dists: latest_local_dist.load_about_data(dest_dir=dest_dir) if not latest_local_dist.has_key_metadata(): @@ -2070,7 +2096,8 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files( + dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get a latest version of the same package that is not our version @@ -2111,7 +2138,8 @@ def get_other_dists(_package, _dist): # if local_dist.has_key_metadata() or not local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir) - lic_errs = local_dist.fetch_license_files(dest_dir, use_cached_index=use_cached_index) + lic_errs = local_dist.fetch_license_files( + dest_dir, use_cached_index=use_cached_index) if not local_dist.has_key_metadata(): print(f"Unable to add essential ABOUT data for: {local_dist}") @@ -2259,7 +2287,8 @@ def find_problems( for dist in package.get_distributions(): dist.load_about_data(dest_dir=dest_dir) - abpth = os.path.abspath(os.path.join(dest_dir, dist.about_filename)) + abpth = os.path.abspath(os.path.join( + dest_dir, dist.about_filename)) if not dist.has_key_metadata(): print(f" Missing key ABOUT data in file://{abpth}") if "classifiers" in dist.extra_data: diff --git a/setup.cfg b/setup.cfg index a8e20c5d..ef7d369b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,7 +6,7 @@ license = Apache-2.0 description = skeleton long_description = file:README.rst long_description_content_type = text/x-rst -url = https://github.com/nexB/skeleton +url = https://github.com/aboutcode-org/skeleton author = nexB. Inc. and others author_email = info@aboutcode.org diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py index 2eb6e558..b4ce8c16 100644 --- a/tests/test_skeleton_codestyle.py +++ b/tests/test_skeleton_codestyle.py @@ -3,7 +3,7 @@ # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # From 9825f6df405ddc1578720b1ae47e8596cca8454e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 4 Sep 2024 15:28:34 +0200 Subject: [PATCH 015/120] Add new arg to gen missing test files This is useful when creating new test suites for copyrights Signed-off-by: Philippe Ombredanne --- src/commoncode/testcase.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/commoncode/testcase.py b/src/commoncode/testcase.py index 3fe61445..6ce7b582 100644 --- a/src/commoncode/testcase.py +++ b/src/commoncode/testcase.py @@ -306,8 +306,7 @@ def make_non_executable(location): current_stat = stat.S_IMODE(os.lstat(location).st_mode) os.chmod(location, current_stat & ~stat.S_IEXEC) - -def get_test_file_pairs(test_dir): +def get_test_file_pairs(test_dir, template_to_generate_missing_yaml=None): """ Yield tuples of (data_file, test_file) from a test data `test_dir` directory. Raise exception for orphaned/dangling files. From 8da28b74eaab7b0fa6c343860e87847b5e3f9d3e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 5 Sep 2024 01:08:16 +0200 Subject: [PATCH 016/120] Update CHANGELOG Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1b731573..9b6b6950 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,13 @@ Release notes ============= +Version 31.2.2 - (2024-09-05) +----------------------------- + +- Add new optional argument to generate test data files from a template +- Migrate URLs to new aboutcode.org org + + Version 31.2.1 - (2024-05-16) ----------------------------- From 325874ca40ff5b9a44da8e38c975964834aa5f3c Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 5 Sep 2024 01:19:19 +0200 Subject: [PATCH 017/120] Drop Python 3.7 Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 3 ++- azure-pipelines.yml | 18 +++++++++--------- setup.cfg | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9b6b6950..d1c440a5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,11 +1,12 @@ Release notes ============= -Version 31.2.2 - (2024-09-05) +Version 32.0.0 - (2024-09-05) ----------------------------- - Add new optional argument to generate test data files from a template - Migrate URLs to new aboutcode.org org +- Drop support for Python 3.7 Version 31.2.1 - (2024-05-16) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 59ab431b..08bc2baf 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,7 +13,7 @@ jobs: parameters: job_name: ubuntu20_cpython image_name: ubuntu-20.04 - python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -21,7 +21,7 @@ jobs: parameters: job_name: ubuntu22_cpython image_name: ubuntu-22.04 - python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -29,7 +29,7 @@ jobs: parameters: job_name: macos12_cpython image_name: macOS-12 - python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -37,7 +37,7 @@ jobs: parameters: job_name: macos13_cpython image_name: macOS-13 - python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -45,7 +45,7 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -53,7 +53,7 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -65,7 +65,7 @@ jobs: parameters: job_name: ubuntu20_test_all_supported_click_versions image_name: ubuntu-20.04 - python_versions: ["3.7", "3.8", "3.9", "3.10"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: click_versions: | for clk_ver in 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7; @@ -82,7 +82,7 @@ jobs: parameters: job_name: ubuntu20_cpython_latest_from_pip image_name: ubuntu-20.04 - python_versions: ["3.7", "3.8", "3.9", "3.10"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: all: venv/bin/pip install --upgrade-strategy eager --force-reinstall @@ -92,7 +92,7 @@ jobs: parameters: job_name: win2019_cpython_latest_from_pip image_name: windows-2019 - python_versions: ["3.7", "3.8", "3.9", "3.10"] + python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: all: venv\Scripts\pip install --upgrade-strategy eager --force-reinstall diff --git a/setup.cfg b/setup.cfg index d7361f83..74208956 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ zip_safe = false setup_requires = setuptools_scm[toml] >= 4 -python_requires = >=3.7 +python_requires = >=3.8 install_requires = attrs >= 18.1, !=20.1.0 From 1e65c5b5fba328fa5e87d3bf347496587484fc25 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 5 Sep 2024 09:52:24 +0200 Subject: [PATCH 018/120] Add debug tracing to CI test Adding a pip freeze to find which dependencies changed significantly Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 08bc2baf..d5510e6d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -84,9 +84,10 @@ jobs: image_name: ubuntu-20.04 python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: - all: - venv/bin/pip install --upgrade-strategy eager --force-reinstall - --upgrade -e . && venv/bin/pytest -n 2 -vvs + all: | + venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e . + venv/bin/pip freeze + venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-win.yml parameters: @@ -94,6 +95,7 @@ jobs: image_name: windows-2019 python_versions: ["3.8", "3.9", "3.10", "3.11"] test_suites: - all: - venv\Scripts\pip install --upgrade-strategy eager --force-reinstall - --upgrade -e . && venv\Scripts\pytest -n 2 -vvs + all: | + venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e . + venv\Scripts\pip freeze + venv\Scripts\pytest -n 2 -vvs From 12031727cd196ec3182ac183dc48d1e4eaf3df8b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 5 Sep 2024 11:56:52 +0200 Subject: [PATCH 019/120] Switch to chardet for stable behavior charset_normalizer has stability issues Reference: https://github.com/jawah/charset_normalizer/issues/520 Signed-off-by: Philippe Ombredanne --- requirements.txt | 2 +- setup.cfg | 4 ++-- tests/test_paths.py | 7 +++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2be5ed22..e4b25aa3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ attrs==22.2.0 beautifulsoup4==4.11.2 certifi==2022.12.7 -charset-normalizer==3.0.1 +chardet==5.2.0 click==8.1.3 idna==3.4 pip==23.0 diff --git a/setup.cfg b/setup.cfg index 74208956..163d21b4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,9 +41,9 @@ python_requires = >=3.8 install_requires = attrs >= 18.1, !=20.1.0 - Beautifulsoup4 >= 4.0.0 + Beautifulsoup4[chardet] >= 4.0.0 click >= 6.7, !=7.0 - requests >= 2.7.0 + requests[use_chardet_on_py3] >= 2.7.0 saneyaml >= 0.5.2 text_unidecode >= 1.0 diff --git a/tests/test_paths.py b/tests/test_paths.py index 5d7d94d2..c9ca8ffa 100644 --- a/tests/test_paths.py +++ b/tests/test_paths.py @@ -76,8 +76,11 @@ def test_safe_path_posix_style_french_char(self): def test_safe_path_posix_style_chinese_char(self): test = paths.safe_path(b'/includes/webform.compon\xd2\xaants.inc/') - expected = 'includes/webform.componNSnts.inc' - assert test == expected + expected = [ + 'includes/webform.componNSnts.inc', + 'includes/webform.componS_nts.inc', + ] + assert test in expected def test_safe_path_windows_style_dots(self): test = paths.safe_path('\\includes\\..\\webform.components.inc\\') From 38c6f22254f6346b807f6b3d94c50bae9a268842 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 5 Sep 2024 11:57:30 +0200 Subject: [PATCH 020/120] Adopt black and isort style Signed-off-by: Philippe Ombredanne --- src/commoncode/__init__.py | 10 +- src/commoncode/archive.py | 16 +- src/commoncode/cliutils.py | 205 +++--- src/commoncode/codec.py | 16 +- src/commoncode/command.py | 81 ++- src/commoncode/compat.py | 5 +- src/commoncode/datautils.py | 127 +++- src/commoncode/date.py | 9 +- src/commoncode/dict_utils.py | 1 - src/commoncode/distro.py | 10 +- src/commoncode/fetch.py | 9 +- src/commoncode/fileset.py | 36 +- src/commoncode/filetype.py | 60 +- src/commoncode/fileutils.py | 94 ++- src/commoncode/functional.py | 6 +- src/commoncode/hash.py | 47 +- src/commoncode/ignore.py | 381 +++++----- src/commoncode/paths.py | 121 ++-- src/commoncode/resource.py | 393 +++++----- src/commoncode/system.py | 62 +- src/commoncode/testcase.py | 116 +-- src/commoncode/text.py | 36 +- src/commoncode/timeutils.py | 10 +- src/commoncode/urn.py | 42 +- src/commoncode/version.py | 44 +- tests/test_cliutils.py | 97 +-- tests/test_cliutils_progressbar.py | 5 +- tests/test_codec.py | 29 +- tests/test_command.py | 160 ++-- tests/test_date.py | 17 +- tests/test_distro.py | 8 +- tests/test_fileset.py | 100 +-- tests/test_filetype.py | 133 ++-- tests/test_fileutils.py | 719 +++++++++--------- tests/test_functional.py | 11 +- tests/test_hash.py | 162 +++-- tests/test_ignore.py | 212 +++--- tests/test_paths.py | 258 ++++--- tests/test_resource.py | 1086 +++++++++++++--------------- tests/test_saneyaml.py | 10 +- tests/test_skeleton_codestyle.py | 2 +- tests/test_text.py | 54 +- tests/test_timeutils.py | 175 ++++- tests/test_urn.py | 96 ++- tests/test_version.py | 201 +++-- 45 files changed, 2832 insertions(+), 2640 deletions(-) diff --git a/src/commoncode/__init__.py b/src/commoncode/__init__.py index 29888e4d..79584fca 100644 --- a/src/commoncode/__init__.py +++ b/src/commoncode/__init__.py @@ -14,16 +14,16 @@ def set_re_max_cache(max_cache=1000000): libraries use a lot of regexes: therefore 100 is not enough to benefit from caching. """ - import re import fnmatch + import re - remax = getattr(re, '_MAXCACHE', 0) + remax = getattr(re, "_MAXCACHE", 0) if remax < max_cache: - setattr(re, '_MAXCACHE', max_cache) + setattr(re, "_MAXCACHE", max_cache) - fnmatchmax = getattr(fnmatch, '_MAXCACHE', 0) + fnmatchmax = getattr(fnmatch, "_MAXCACHE", 0) if fnmatchmax < max_cache: - setattr(fnmatch, '_MAXCACHE', max_cache) + setattr(fnmatch, "_MAXCACHE", max_cache) set_re_max_cache() diff --git a/src/commoncode/archive.py b/src/commoncode/archive.py index 671c4517..3c0c7583 100644 --- a/src/commoncode/archive.py +++ b/src/commoncode/archive.py @@ -6,12 +6,12 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -from functools import partial -import os -from os import path import gzip +import os import tarfile import zipfile +from functools import partial +from os import path from commoncode.system import on_windows @@ -47,7 +47,7 @@ def extract_tar(location, target_dir, verbatim=False, *args, **kwargs): # always for using bytes for paths on all OSses... tar seems to use bytes internally # and get confused otherwise location = os.fsencode(location) - with open(location, 'rb') as input_tar: + with open(location, "rb") as input_tar: tar = None try: tar = tarfile.open(fileobj=input_tar) @@ -69,7 +69,7 @@ def extract_zip(location, target_dir, *args, **kwargs): Extract a zip archive file at location in the target_dir directory. """ if not path.isfile(location) and zipfile.is_zipfile(location): - raise Exception('Incorrect zip file %(location)r' % locals()) + raise Exception("Incorrect zip file %(location)r" % locals()) with zipfile.ZipFile(location) as zipf: for info in zipf.infolist(): @@ -82,7 +82,7 @@ def extract_zip(location, target_dir, *args, **kwargs): if not path.exists(target): os.makedirs(target) if not path.exists(target): - with open(target, 'wb') as f: + with open(target, "wb") as f: f.write(content) @@ -92,7 +92,7 @@ def extract_zip_raw(location, target_dir, *args, **kwargs): Use the builtin extractall function """ if not path.isfile(location) and zipfile.is_zipfile(location): - raise Exception('Incorrect zip file %(location)r' % locals()) + raise Exception("Incorrect zip file %(location)r" % locals()) with zipfile.ZipFile(location) as zipf: zipf.extractall(path=target_dir) @@ -124,6 +124,6 @@ def get_gz_compressed_file_content(location): Uncompress a compressed file at `location` and return its content as a byte string. Raise Exceptions on errors. """ - with gzip.GzipFile(location, 'rb') as compressed: + with gzip.GzipFile(location, "rb") as compressed: content = compressed.read() return content diff --git a/src/commoncode/cliutils.py b/src/commoncode/cliutils.py index b58736fc..1638088d 100644 --- a/src/commoncode/cliutils.py +++ b/src/commoncode/cliutils.py @@ -11,11 +11,11 @@ import click -from click.utils import echo -from click.termui import style -from click.types import BoolParamType # FIXME: this is NOT API from click._termui_impl import ProgressBar +from click.termui import style +from click.types import BoolParamType +from click.utils import echo from commoncode.fileutils import file_name from commoncode.fileutils import splitext @@ -31,13 +31,14 @@ def logger_debug(*args): if TRACE: import logging + logger = logging.getLogger(__name__) logging.basicConfig(stream=sys.stdout) logger.setLevel(logging.DEBUG) def logger_debug(*args): - return logger.debug(' '.join(isinstance(a, str) - and a or repr(a) for a in args)) + return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args)) + """ Command line UI utilities for improved options, help and progress reporting. @@ -51,7 +52,7 @@ class BaseCommand(click.Command): # override this in sub-classes with a command-specific message such as # "Try 'scancode --help' for help on options and arguments." - short_usage_help = '' + short_usage_help = "" def get_usage(self, ctx): """ @@ -61,8 +62,12 @@ def get_usage(self, ctx): return super(BaseCommand, self).get_usage(ctx) + self.short_usage_help def main( - self, args=None, prog_name=None, complete_var=None, - standalone_mode=True, **extra, + self, + args=None, + prog_name=None, + complete_var=None, + standalone_mode=True, + **extra, ): """ Workaround click 4.0 bug https://github.com/mitsuhiko/click/issues/365 @@ -83,14 +88,20 @@ class GroupedHelpCommand(BaseCommand): help where each option is grouped by group in the help. """ - short_usage_help = ''' -Try the '--help' option for help on options and arguments.''' + short_usage_help = """ +Try the '--help' option for help on options and arguments.""" def __init__( - self, name, context_settings=None, callback=None, params=None, + self, + name, + context_settings=None, + callback=None, + params=None, help=None, # NOQA - epilog=None, short_help=None, - options_metavar='[OPTIONS]', add_help_option=True, + epilog=None, + short_help=None, + options_metavar="[OPTIONS]", + add_help_option=True, plugin_options=(), ): """ @@ -120,19 +131,21 @@ def format_options(self, ctx, formatter): to MISC_GROUP group. """ # this mapping defines the CLI help presentation order - help_groups = dict([ - (SCAN_GROUP, []), - (OTHER_SCAN_GROUP, []), - (SCAN_OPTIONS_GROUP, []), - (OUTPUT_GROUP, []), - (OUTPUT_FILTER_GROUP, []), - (OUTPUT_CONTROL_GROUP, []), - (PRE_SCAN_GROUP, []), - (POST_SCAN_GROUP, []), - (CORE_GROUP, []), - (MISC_GROUP, []), - (DOC_GROUP, []), - ]) + help_groups = dict( + [ + (SCAN_GROUP, []), + (OTHER_SCAN_GROUP, []), + (SCAN_OPTIONS_GROUP, []), + (OUTPUT_GROUP, []), + (OUTPUT_FILTER_GROUP, []), + (OUTPUT_CONTROL_GROUP, []), + (PRE_SCAN_GROUP, []), + (POST_SCAN_GROUP, []), + (CORE_GROUP, []), + (MISC_GROUP, []), + (DOC_GROUP, []), + ] + ) for param in self.get_params(ctx): # Get the list of option's name and help text @@ -140,11 +153,11 @@ def format_options(self, ctx, formatter): if not help_record: continue # organize options by group - help_group = getattr(param, 'help_group', MISC_GROUP) - sort_order = getattr(param, 'sort_order', 100) + help_group = getattr(param, "help_group", MISC_GROUP) + sort_order = getattr(param, "sort_order", 100) help_groups[help_group].append((sort_order, help_record)) - with formatter.section('Options'): + with formatter.section("Options"): for group, help_records in help_groups.items(): if not help_records: continue @@ -156,7 +169,6 @@ def format_options(self, ctx, formatter): # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 class DebuggedProgressBar(ProgressBar): - # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 def make_step(self, n_steps): @@ -221,7 +233,7 @@ def format_progress_line(self): if self.item_show_func: item_info = self.item_show_func(self.current_item) else: - item_info = '.' + item_info = "." if item_info: return item_info @@ -230,16 +242,16 @@ def render_finish(self): BAR_WIDTH = 20 -BAR_SEP_LEN = len(' ') +BAR_SEP_LEN = len(" ") def progressmanager( iterable=None, length=None, - fill_char='#', - empty_char='-', + fill_char="#", + empty_char="-", bar_template=None, - info_sep=' ', + info_sep=" ", show_eta=False, show_percent=False, show_pos=True, @@ -264,10 +276,7 @@ def progressmanager( progress_class = ProgressLogger else: progress_class = EnhancedProgressBar - bar_template = ( - '[%(bar)s]' + ' ' + '%(info)s' - if bar_template is None else bar_template - ) + bar_template = "[%(bar)s]" + " " + "%(info)s" if bar_template is None else bar_template kwargs = dict( iterable=iterable, @@ -293,8 +302,8 @@ def progressmanager( # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 # Here we create a dummy progress_class and then for the attribute presence. pb = progress_class([]) - if hasattr(pb, 'update_min_steps'): - kwargs['update_min_steps'] = update_min_steps + if hasattr(pb, "update_min_steps"): + kwargs["update_min_steps"] = update_min_steps return progress_class(**kwargs) @@ -317,7 +326,7 @@ def fixed_width_file_name(path, max_length=25): >>> assert fwfn == '' """ if not path: - return '' + return "" # get the path as unicode for display! filename = file_name(path) @@ -329,13 +338,13 @@ def fixed_width_file_name(path, max_length=25): remaining_length = max_length - len_ext - dots if remaining_length < 5 or remaining_length < (len_ext + dots): - return '' + return "" prefix_and_suffix_length = abs(remaining_length // 2) prefix = base_name[:prefix_and_suffix_length] - ellipsis = dots * '.' + ellipsis = dots * "." suffix = base_name[-prefix_and_suffix_length:] - return '{prefix}{ellipsis}{suffix}{ext}'.format(**locals()) + return "{prefix}{ellipsis}{suffix}{ext}".format(**locals()) def file_name_max_len(used_width=BAR_WIDTH + 1 + 7 + 1 + 8 + 1): @@ -359,13 +368,13 @@ def file_name_max_len(used_width=BAR_WIDTH + 1 + 7 + 1 + 8 + 1): return max_filename_length -def path_progress_message(item, verbose=False, prefix='Scanned: '): +def path_progress_message(item, verbose=False, prefix="Scanned: "): """ Return a styled message suitable for progress display when processing a path for an `item` tuple of (location, rid, scan_errors, *other items) """ if not item: - return '' + return "" location = item[0] errors = item[2] location = toascii(location) @@ -374,25 +383,25 @@ def path_progress_message(item, verbose=False, prefix='Scanned: '): max_file_name_len = file_name_max_len() # do not display a file name in progress bar if there is no space available if max_file_name_len <= 10: - return '' + return "" progress_line = fixed_width_file_name(location, max_file_name_len) - color = 'red' if errors else 'green' + color = "red" if errors else "green" return style(prefix) + style(progress_line, fg=color) # CLI help groups -SCAN_GROUP = 'primary scans' -SCAN_OPTIONS_GROUP = 'scan options' -OTHER_SCAN_GROUP = 'other scans' -OUTPUT_GROUP = 'output formats' -OUTPUT_CONTROL_GROUP = 'output control' -OUTPUT_FILTER_GROUP = 'output filters' -PRE_SCAN_GROUP = 'pre-scan' -POST_SCAN_GROUP = 'post-scan' -MISC_GROUP = 'miscellaneous' -DOC_GROUP = 'documentation' -CORE_GROUP = 'core' +SCAN_GROUP = "primary scans" +SCAN_OPTIONS_GROUP = "scan options" +OTHER_SCAN_GROUP = "other scans" +OUTPUT_GROUP = "output formats" +OUTPUT_CONTROL_GROUP = "output control" +OUTPUT_FILTER_GROUP = "output filters" +PRE_SCAN_GROUP = "pre-scan" +POST_SCAN_GROUP = "post-scan" +MISC_GROUP = "miscellaneous" +DOC_GROUP = "documentation" +CORE_GROUP = "core" class PluggableCommandLineOption(click.Option): @@ -431,7 +440,7 @@ def __init__( # a sequence of other option name strings that this option # conflicts with if they are set conflicting_options=(), - **kwargs + **kwargs, ): super(PluggableCommandLineOption, self).__init__( param_decls=param_decls, @@ -446,7 +455,7 @@ def __init__( allow_from_autoenv=allow_from_autoenv, type=type, help=help, - **kwargs + **kwargs, ) self.help_group = help_group @@ -463,9 +472,9 @@ def __repr__(self, *args, **kwargs): conflicting_options = self.conflicting_options return ( - 'PluggableCommandLineOption' % locals() + "PluggableCommandLineOption" % locals() ) def validate_dependencies(self, ctx, value): @@ -473,10 +482,8 @@ def validate_dependencies(self, ctx, value): Validate `value` against declared `required_options` or `conflicting_options` dependencies. """ - _validate_option_dependencies( - ctx, self, value, self.required_options, required=True) - _validate_option_dependencies( - ctx, self, value, self.conflicting_options, required=False) + _validate_option_dependencies(ctx, self, value, self.required_options, required=True) + _validate_option_dependencies(ctx, self, value, self.conflicting_options, required=False) def get_help_record(self, ctx): if not self.hidden: @@ -490,22 +497,20 @@ def validate_option_dependencies(ctx): """ values = ctx.params if TRACE: - logger_debug('validate_option_dependencies: values:') + logger_debug("validate_option_dependencies: values:") for va in sorted(values.items()): - logger_debug(' ', va) + logger_debug(" ", va) for param in ctx.command.params: if param.is_eager: continue if not isinstance(param, PluggableCommandLineOption): if TRACE: - logger_debug( - ' validate_option_dependencies: skip param:', param) + logger_debug(" validate_option_dependencies: skip param:", param) continue value = values.get(param.name) if TRACE: - logger_debug(' validate_option_dependencies: param:', - param, 'value:', value) + logger_debug(" validate_option_dependencies: param:", param, "value:", value) param.validate_dependencies(ctx, value) @@ -537,8 +542,8 @@ def _is_set(_value, _param): if TRACE: logger_debug() - logger_debug('Checking param:', param) - logger_debug(' value:', value, 'is_set:', is_set) + logger_debug("Checking param:", param) + logger_debug(" value:", value, "is_set:", is_set) if not is_set: return @@ -556,27 +561,29 @@ def _is_set(_value, _param): if TRACE: logger_debug() - logger_debug(' Available other params:') + logger_debug(" Available other params:") for oparam in oparams: - logger_debug(' other param:', oparam) - logger_debug(' value:', ctx.params.get(oparam.name)) + logger_debug(" other param:", oparam) + logger_debug(" value:", ctx.params.get(oparam.name)) if required: - logger_debug(' missing names:', missing_onames) + logger_debug(" missing names:", missing_onames) if required and missing_onames: opt = param.opts[-1] oopts = [oparam.opts[-1] for oparam in oparams] - omopts = ['--' + oname.replace('_', '-') for oname in missing_onames] + omopts = ["--" + oname.replace("_", "-") for oname in missing_onames] oopts.extend(omopts) - oopts = ', '.join(oopts) - msg = ('The option %(opt)s requires the option(s) %(all_opts)s.' - 'and is missing %(omopts)s. ' - 'You must set all of these options if you use this option.' % locals()) + oopts = ", ".join(oopts) + msg = ( + "The option %(opt)s requires the option(s) %(all_opts)s." + "and is missing %(omopts)s. " + "You must set all of these options if you use this option." % locals() + ) raise click.UsageError(msg) if TRACE: logger_debug() - logger_debug(' Checking other params:') + logger_debug(" Checking other params:") opt = param.opts[-1] @@ -585,21 +592,25 @@ def _is_set(_value, _param): ois_set = _is_set(ovalue, oparam) if TRACE: - logger_debug(' Checking oparam:', oparam) - logger_debug(' value:', ovalue, 'ois_set:', ois_set) + logger_debug(" Checking oparam:", oparam) + logger_debug(" value:", ovalue, "ois_set:", ois_set) # by convention the last opt is the long form oopt = oparam.opts[-1] - oopts = ', '.join(oparam.opts[-1] for oparam in oparams) - all_opts = '%(opt)s and %(oopts)s' % locals() + oopts = ", ".join(oparam.opts[-1] for oparam in oparams) + all_opts = "%(opt)s and %(oopts)s" % locals() if required and not ois_set: - msg = ('The option %(opt)s requires the option(s) %(oopts)s ' - 'and is missing %(oopt)s. ' - 'You must set all of these options if you use this option.' % locals()) + msg = ( + "The option %(opt)s requires the option(s) %(oopts)s " + "and is missing %(oopt)s. " + "You must set all of these options if you use this option." % locals() + ) raise click.UsageError(msg) if not required and ois_set: - msg = ('The option %(opt)s cannot be used together with the %(oopts)s option(s) ' - 'and %(oopt)s is used. ' - 'You can set only one of these options at a time.' % locals()) + msg = ( + "The option %(opt)s cannot be used together with the %(oopts)s option(s) " + "and %(oopt)s is used. " + "You can set only one of these options at a time." % locals() + ) raise click.UsageError(msg) diff --git a/src/commoncode/codec.py b/src/commoncode/codec.py index 07070680..fe84d630 100644 --- a/src/commoncode/codec.py +++ b/src/commoncode/codec.py @@ -14,8 +14,12 @@ """ -def c2i(c): return c -def i2c(i): return bytes([i]) +def c2i(c): + return c + + +def i2c(i): + return bytes([i]) def num_to_bin(num): @@ -26,16 +30,16 @@ def num_to_bin(num): """ # Zero is not encoded but returned as an empty value if num == 0: - return b'\x00' + return b"\x00" - return num.to_bytes((num.bit_length() + 7) // 8, 'big') + return num.to_bytes((num.bit_length() + 7) // 8, "big") def bin_to_num(binstr): """ Convert a big endian byte-ordered binary string to an integer or long. """ - return int.from_bytes(binstr, byteorder='big', signed=False) + return int.from_bytes(binstr, byteorder="big", signed=False) def urlsafe_b64encode(s): @@ -50,7 +54,7 @@ def urlsafe_b64decode(b64): Decode a url safe base64-encoded string. Note that we use stddecode to work around a bug in the standard library. """ - b = b64.replace(b'-', b'+').replace(b'_', b'/') + b = b64.replace(b"-", b"+").replace(b"_", b"/") return stddecode(b) diff --git a/src/commoncode/command.py b/src/commoncode/command.py index 053e7fb1..2570a05a 100644 --- a/src/commoncode/command.py +++ b/src/commoncode/command.py @@ -6,20 +6,19 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -import ctypes import contextlib +import ctypes import io -import os -from os import path - import logging +import os import signal import subprocess +from os import path +from commoncode import text from commoncode.fileutils import get_temp_dir from commoncode.system import on_posix from commoncode.system import on_windows -from commoncode import text """ Wrapper for executing external commands in sub-processes which works @@ -33,15 +32,16 @@ if TRACE: import sys + logging.basicConfig(stream=sys.stdout) logger.setLevel(logging.DEBUG) # current directory is the root dir of this library curr_dir = path.dirname(path.dirname(path.abspath(__file__))) -PATH_ENV_VAR = 'PATH' -LD_LIBRARY_PATH = 'LD_LIBRARY_PATH' -DYLD_LIBRARY_PATH = 'DYLD_LIBRARY_PATH' +PATH_ENV_VAR = "PATH" +LD_LIBRARY_PATH = "LD_LIBRARY_PATH" +DYLD_LIBRARY_PATH = "DYLD_LIBRARY_PATH" def execute(cmd_loc, args, cwd=None, env=None, to_files=False, log=TRACE): @@ -68,10 +68,10 @@ def execute(cmd_loc, args, cwd=None, env=None, to_files=False, log=TRACE): cwd = cwd or curr_dir # temp files for stderr and stdout - tmp_dir = get_temp_dir(prefix='cmd-') + tmp_dir = get_temp_dir(prefix="cmd-") - sop = path.join(tmp_dir, 'stdout') - sep = path.join(tmp_dir, 'stderr') + sop = path.join(tmp_dir, "stdout") + sep = path.join(tmp_dir, "stderr") # shell==True is DANGEROUS but we are not running arbitrary commands # though we can execute commands that just happen to be in the path @@ -81,15 +81,15 @@ def execute(cmd_loc, args, cwd=None, env=None, to_files=False, log=TRACE): if log: printer = logger.debug if TRACE else lambda x: print(x) printer( - 'Executing command %(cmd_loc)r as:\n%(full_cmd)r\nwith: env=%(env)r\n' - 'shell=%(shell)r\ncwd=%(cwd)r\nstdout=%(sop)r\nstderr=%(sep)r' - % locals()) + "Executing command %(cmd_loc)r as:\n%(full_cmd)r\nwith: env=%(env)r\n" + "shell=%(shell)r\ncwd=%(cwd)r\nstdout=%(sop)r\nstderr=%(sep)r" % locals() + ) proc = None rc = 100 try: - with io.open(sop, 'wb') as stdout, io.open(sep, 'wb') as stderr, pushd(cmd_dir): + with io.open(sop, "wb") as stdout, io.open(sep, "wb") as stderr, pushd(cmd_dir): proc = subprocess.Popen( full_cmd, cwd=cwd, @@ -108,11 +108,11 @@ def execute(cmd_loc, args, cwd=None, env=None, to_files=False, log=TRACE): if not to_files: # return output as ASCII string loaded from the output files - with open(sop, 'rb') as so: + with open(sop, "rb") as so: sor = so.read() sop = text.toascii(sor).strip() - with open(sep, 'rb') as se: + with open(sep, "rb") as se: ser = se.read() sep = text.toascii(ser).strip() @@ -167,18 +167,15 @@ def get_env(base_vars=None, lib_dir=None): # Create and add LD environment variables if lib_dir and on_posix: - new_path = f'{lib_dir}' + new_path = f"{lib_dir}" # on Linux/posix ld_lib_path = os.environ.get(LD_LIBRARY_PATH) - env_vars.update( - {LD_LIBRARY_PATH: update_path_var(ld_lib_path, new_path)}) + env_vars.update({LD_LIBRARY_PATH: update_path_var(ld_lib_path, new_path)}) # on Mac, though LD_LIBRARY_PATH should work too dyld_lib_path = os.environ.get(DYLD_LIBRARY_PATH) - env_vars.update( - {DYLD_LIBRARY_PATH: update_path_var(dyld_lib_path, new_path)}) + env_vars.update({DYLD_LIBRARY_PATH: update_path_var(dyld_lib_path, new_path)}) - env_vars = {text.as_unicode(k): text.as_unicode(v) - for k, v in env_vars.items()} + env_vars = {text.as_unicode(k): text.as_unicode(v) for k, v in env_vars.items()} return env_vars @@ -198,9 +195,9 @@ def close_pipe(p): except IOError: pass - close_pipe(getattr(proc, 'stdin', None)) - close_pipe(getattr(proc, 'stdout', None)) - close_pipe(getattr(proc, 'stderr', None)) + close_pipe(getattr(proc, "stdin", None)) + close_pipe(getattr(proc, "stdout", None)) + close_pipe(getattr(proc, "stderr", None)) try: # Ensure process death otherwise proc.wait may hang in some cases @@ -219,8 +216,7 @@ def load_shared_library(dll_loc, *args): Return the loaded shared library object from the ``dll_loc`` location. """ if not dll_loc or not path.exists(dll_loc): - raise ImportError( - f'Shared library does not exists: dll_loc: {dll_loc}') + raise ImportError(f"Shared library does not exists: dll_loc: {dll_loc}") if not isinstance(dll_loc, str): dll_loc = os.fsdecode(dll_loc) @@ -232,19 +228,22 @@ def load_shared_library(dll_loc, *args): with pushd(dll_dir): lib = ctypes.CDLL(dll_loc) except OSError as e: - from pprint import pformat import traceback - msgs = tuple([ - f'ctypes.CDLL("{dll_loc}")', - 'os.environ:\n{}'.format(pformat(dict(os.environ))), - traceback.format_exc(), - ]) + from pprint import pformat + + msgs = tuple( + [ + f'ctypes.CDLL("{dll_loc}")', + "os.environ:\n{}".format(pformat(dict(os.environ))), + traceback.format_exc(), + ] + ) raise Exception(msgs) from e if lib and lib._name: return lib - raise Exception(f'Failed to load shared library with ctypes: {dll_loc}') + raise Exception(f"Failed to load shared library with ctypes: {dll_loc}") @contextlib.contextmanager @@ -271,7 +270,7 @@ def update_path_var(existing_path_var, new_path): if not new_path: return existing_path_var - existing_path_var = existing_path_var or '' + existing_path_var = existing_path_var or "" existing_path_var = os.fsdecode(existing_path_var) new_path = os.fsdecode(new_path) @@ -296,7 +295,11 @@ def update_path_var(existing_path_var, new_path): return updated_path_var -PATH_VARS = DYLD_LIBRARY_PATH, LD_LIBRARY_PATH, 'PATH', +PATH_VARS = ( + DYLD_LIBRARY_PATH, + LD_LIBRARY_PATH, + "PATH", +) def searchable_paths(env_vars=PATH_VARS): @@ -306,7 +309,7 @@ def searchable_paths(env_vars=PATH_VARS): """ dirs = [] for env_var in env_vars: - value = os.environ.get(env_var, '') or '' + value = os.environ.get(env_var, "") or "" dirs.extend(value.split(os.pathsep)) dirs = [os.path.realpath(d.strip()) for d in dirs if d.strip()] return tuple(d for d in dirs if os.path.isdir(d)) diff --git a/src/commoncode/compat.py b/src/commoncode/compat.py index 001464c7..bad5d38c 100644 --- a/src/commoncode/compat.py +++ b/src/commoncode/compat.py @@ -8,4 +8,7 @@ unicode = str # NOQA long = int # NOQA -integer_types = int, long, +integer_types = ( + int, + long, +) diff --git a/src/commoncode/datautils.py b/src/commoncode/datautils.py index ea71f9aa..db67e219 100644 --- a/src/commoncode/datautils.py +++ b/src/commoncode/datautils.py @@ -6,22 +6,32 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import typing + import attr from attr.validators import in_ as choices # NOQA -import typing """ Utilities and helpers for data classes. """ -HELP_METADATA = '__field_help' -LABEL_METADATA = '__field_label' +HELP_METADATA = "__field_help" +LABEL_METADATA = "__field_label" -def attribute(default=attr.NOTHING, validator=None, - repr=False, eq=True, order=True, # NOQA - init=True, type=None, converter=None, # NOQA - help=None, label=None, metadata=None,): # NOQA +def attribute( + default=attr.NOTHING, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + init=True, + type=None, + converter=None, # NOQA + help=None, + label=None, + metadata=None, +): # NOQA """ A generic attribute with help metadata and that is not included in the representation by default. @@ -42,12 +52,20 @@ def attribute(default=attr.NOTHING, validator=None, init=init, metadata=metadata, type=type, - converter=converter + converter=converter, ) -def Boolean(default=False, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def Boolean( + default=False, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A boolean attribute. """ @@ -65,8 +83,16 @@ def Boolean(default=False, validator=None, repr=False, eq=True, order=True, # N ) -def TriBoolean(default=None, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def TriBoolean( + default=None, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A tri-boolean attribute with possible values of None, True and False. """ @@ -84,8 +110,16 @@ def TriBoolean(default=None, validator=None, repr=False, eq=True, order=True, # ) -def String(default=None, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def String( + default=None, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A string attribute. """ @@ -103,8 +137,16 @@ def String(default=None, validator=None, repr=False, eq=True, order=True, # NOQ ) -def Integer(default=0, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def Integer( + default=0, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ An integer attribute. """ @@ -123,8 +165,16 @@ def Integer(default=0, validator=None, repr=False, eq=True, order=True, # NOQA ) -def Float(default=0.0, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def Float( + default=0.0, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A float attribute. """ @@ -142,9 +192,17 @@ def Float(default=0.0, validator=None, repr=False, eq=True, order=True, # NOQA ) -def List(item_type=typing.Any, default=attr.NOTHING, validator=None, - repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def List( + item_type=typing.Any, + default=attr.NOTHING, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A list attribute: the optional item_type defines the type of items it stores. """ @@ -165,9 +223,17 @@ def List(item_type=typing.Any, default=attr.NOTHING, validator=None, ) -def Mapping(value_type=typing.Any, default=attr.NOTHING, validator=None, - repr=False, eq=True, order=True, # NOQA - converter=None, help=None, label=None): # NOQA +def Mapping( + value_type=typing.Any, + default=attr.NOTHING, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + help=None, + label=None, +): # NOQA """ A mapping attribute: the optional value_type defines the type of values it stores. The key is always a string. @@ -191,13 +257,22 @@ def Mapping(value_type=typing.Any, default=attr.NOTHING, validator=None, label=label, ) + ################################################## # FIXME: add proper support for dates!!! ################################################## -def Date(default=None, validator=None, repr=False, eq=True, order=True, # NOQA - converter=None, label=None, help=None,): # NOQA +def Date( + default=None, + validator=None, + repr=False, + eq=True, + order=True, # NOQA + converter=None, + label=None, + help=None, +): # NOQA """ A date attribute. It always serializes to an ISO date string. Behavior is TBD and for now this is exactly a string. diff --git a/src/commoncode/date.py b/src/commoncode/date.py index 81da3d0d..2493265e 100644 --- a/src/commoncode/date.py +++ b/src/commoncode/date.py @@ -7,12 +7,12 @@ # import calendar -from datetime import datetime import os +from datetime import datetime def isoformat(utc_date): - return datetime.isoformat(utc_date).replace('T', ' ') + return datetime.isoformat(utc_date).replace("T", " ") def get_file_mtime(location, iso=True): @@ -20,7 +20,7 @@ def get_file_mtime(location, iso=True): Return a string containing the last modified date of a file formatted as an ISO time stamp if ISO is True or as a raw number since epoch. """ - date = '' + date = "" # FIXME: use file types if not os.path.isdir(location): mtime = os.stat(location).st_mtime @@ -37,5 +37,4 @@ def secs_from_epoch(d): Return a number of seconds since epoch for a date time stamp """ # FIXME: what does this do? - return calendar.timegm(datetime.strptime(d.split('.')[0], - '%Y-%m-%d %H:%M:%S').timetuple()) + return calendar.timegm(datetime.strptime(d.split(".")[0], "%Y-%m-%d %H:%M:%S").timetuple()) diff --git a/src/commoncode/dict_utils.py b/src/commoncode/dict_utils.py index ca017d56..1473f6da 100644 --- a/src/commoncode/dict_utils.py +++ b/src/commoncode/dict_utils.py @@ -1,4 +1,3 @@ - # Copyright (c) 2003-2012 Raymond Hettinger # SPDX-License-Identifier: Python-2.0 diff --git a/src/commoncode/distro.py b/src/commoncode/distro.py index 3b22354d..2e1a29a0 100644 --- a/src/commoncode/distro.py +++ b/src/commoncode/distro.py @@ -32,11 +32,5 @@ def parse_os_release(location): """ with open(location) as osrl: lines = (line.strip() for line in osrl) - lines = ( - line.partition('=') for line in lines - if line and not line.startswith('#') - ) - return { - key.strip(): ''.join(shlex.split(value)) - for key, _, value in lines - } + lines = (line.partition("=") for line in lines if line and not line.startswith("#")) + return {key.strip(): "".join(shlex.split(value)) for key, _, value in lines} diff --git a/src/commoncode/fetch.py b/src/commoncode/fetch.py index 9428e32f..3db98c27 100644 --- a/src/commoncode/fetch.py +++ b/src/commoncode/fetch.py @@ -35,19 +35,18 @@ def download_url(url, file_name=None, verify=True, timeout=10): try: response = requests.get(url, **requests_args) except (ConnectionError, InvalidSchema) as e: - logger.error('download_url: Download failed for %(url)r' % locals()) + logger.error("download_url: Download failed for %(url)r" % locals()) raise status = response.status_code if status != 200: - msg = 'download_url: Download failed for %(url)r with %(status)r' % locals( - ) + msg = "download_url: Download failed for %(url)r with %(status)r" % locals() logger.error(msg) raise Exception(msg) - tmp_dir = fileutils.get_temp_dir(prefix='fetch-') + tmp_dir = fileutils.get_temp_dir(prefix="fetch-") output_file = os.path.join(tmp_dir, file_name) - with open(output_file, 'wb') as out: + with open(output_file, "wb") as out: out.write(response.content) return output_file diff --git a/src/commoncode/fileset.py b/src/commoncode/fileset.py index 99323f83..9ebab26b 100644 --- a/src/commoncode/fileset.py +++ b/src/commoncode/fileset.py @@ -83,16 +83,14 @@ def is_included(path, includes=None, excludes=None): if includes: included = get_matches(path, includes, all_matches=False) if TRACE: - logger.debug( - 'in_fileset: path: %(path)r included:%(included)r' % locals()) + logger.debug("in_fileset: path: %(path)r included:%(included)r" % locals()) if not included: return False if excludes: excluded = get_matches(path, excludes, all_matches=False) if TRACE: - logger.debug( - 'in_fileset: path: %(path)r excluded:%(excluded)r .' % locals()) + logger.debug("in_fileset: path: %(path)r excluded:%(excluded)r ." % locals()) if excluded: return False @@ -110,41 +108,39 @@ def get_matches(path, patterns, all_matches=False): return False path = fileutils.as_posixpath(path).lower() - pathstripped = path.lstrip('/0') + pathstripped = path.lstrip("/0") if not pathstripped: return False segments = paths.split(pathstripped) if TRACE: - logger.debug( - '_match: path: %(path)r patterns:%(patterns)r.' % locals()) + logger.debug("_match: path: %(path)r patterns:%(patterns)r." % locals()) matches = [] if not isinstance(patterns, dict): - assert isinstance(patterns, (list, tuple) - ), 'Invalid patterns: {}'.format(patterns) + assert isinstance(patterns, (list, tuple)), "Invalid patterns: {}".format(patterns) patterns = {p: p for p in patterns} for pat, value in patterns.items(): if not pat or not pat.strip(): continue - value = value or '' - pat = pat.lstrip('/').lower() - is_plain = '/' not in pat + value = value or "" + pat = pat.lstrip("/").lower() + is_plain = "/" not in pat if is_plain: if any(fnmatch.fnmatchcase(s, pat) for s in segments): matches.append(value) if not all_matches: break - elif (fnmatch.fnmatchcase(path, pat) or fnmatch.fnmatchcase(pathstripped, pat)): + elif fnmatch.fnmatchcase(path, pat) or fnmatch.fnmatchcase(pathstripped, pat): matches.append(value) if not all_matches: break if TRACE: - logger.debug('_match: matches: %(matches)r' % locals()) + logger.debug("_match: matches: %(matches)r" % locals()) if not all_matches: if matches: @@ -161,9 +157,9 @@ def load(location): if not location: return tuple() fn = os.path.abspath(os.path.normpath(os.path.expanduser(location))) - msg = ('File %(location)s does not exist or not a file.') % locals() - assert (os.path.exists(fn) and os.path.isfile(fn)), msg - mode = 'r' + msg = ("File %(location)s does not exist or not a file.") % locals() + assert os.path.exists(fn) and os.path.isfile(fn), msg + mode = "r" with open(fn, mode) as f: return [l.strip() for l in f if l and l.strip()] @@ -175,9 +171,9 @@ def includes_excludes(patterns, message): value in the returned mappings. Ignore pattern as comments if prefixed with #. Use an empty string is message is None. """ - message = message or '' - BANG = '!' - POUND = '#' + message = message or "" + BANG = "!" + POUND = "#" included = {} excluded = {} if not patterns: diff --git a/src/commoncode/filetype.py b/src/commoncode/filetype.py index 04d82105..a7681d4f 100644 --- a/src/commoncode/filetype.py +++ b/src/commoncode/filetype.py @@ -7,10 +7,11 @@ # import os -from datetime import datetime, timezone +from datetime import datetime +from datetime import timezone -from commoncode.system import on_posix from commoncode.functional import memoize +from commoncode.system import on_posix """ Low level file type utilities, essentially a wrapper around os.path and stat. @@ -77,7 +78,7 @@ def get_link_target(location): Return the link target for `location` if this is a Link or an empty string. """ - target = '' + target = "" # always false on windows, until Python supports junctions/links if on_posix and is_link(location): try: @@ -91,12 +92,38 @@ def get_link_target(location): # Map of type checker function -> short type code # The order of types check matters: link -> file -> directory -> special -TYPES = dict([ - (is_link, ('l', 'link',)), - (is_file, ('f', 'file',)), - (is_dir, ('d', 'directory',)), - (is_special, ('s', 'special',)) -]) +TYPES = dict( + [ + ( + is_link, + ( + "l", + "link", + ), + ), + ( + is_file, + ( + "f", + "file", + ), + ), + ( + is_dir, + ( + "d", + "directory", + ), + ), + ( + is_special, + ( + "s", + "special", + ), + ), + ] +) def get_type(location, short=True): @@ -161,7 +188,7 @@ def get_last_modified_date(location): Return the last modified date stamp of a file as YYYYMMDD format. The date of non-files (dir, links, special) is always an empty string. """ - yyyymmdd = '' + yyyymmdd = "" if is_file(location): utc_date = datetime.isoformat( datetime.fromtimestamp( @@ -174,8 +201,8 @@ def get_last_modified_date(location): counting_functions = { - 'file_count': lambda _: 1, - 'file_size': os.path.getsize, + "file_count": lambda _: 1, + "file_size": os.path.getsize, } @@ -205,8 +232,9 @@ def counter(location, counting_function): count_fun = counting_functions[counting_function] return count_fun(location) elif is_dir(location): - count += sum(counter(os.path.join(location, p), counting_function) - for p in os.listdir(location)) + count += sum( + counter(os.path.join(location, p), counting_function) for p in os.listdir(location) + ) return count @@ -216,7 +244,7 @@ def get_file_count(location): or 1 if `location` is a file. Only regular files are counted. Everything else has a zero size. """ - return counter(location, 'file_count') + return counter(location, "file_count") def get_size(location): @@ -225,4 +253,4 @@ def get_size(location): directory, the cumulative size of all files in this directory tree. Only regular files have a size. Everything else has a zero size. """ - return counter(location, 'file_size') + return counter(location, "file_size") diff --git a/src/commoncode/fileutils.py b/src/commoncode/fileutils.py index d0aa38b5..123a7175 100644 --- a/src/commoncode/fileutils.py +++ b/src/commoncode/fileutils.py @@ -7,14 +7,13 @@ # import errno -import os import ntpath +import os import posixpath import shutil import stat import sys import tempfile - from os import fsdecode try: @@ -33,7 +32,9 @@ class WindowsError(Exception): pass + import logging + logger = logging.getLogger(__name__) TRACE = False @@ -48,7 +49,8 @@ def logger_debug(*args): logger.setLevel(logging.DEBUG) def logger_debug(*args): - return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args)) + return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args)) + """ File, paths and directory utility functions. @@ -68,8 +70,7 @@ def create_dir(location): if os.path.exists(location): if not os.path.isdir(location): - err = ('Cannot create directory: existing file ' - 'in the way ''%(location)s.') + err = "Cannot create directory: existing file " "in the way " "%(location)s." raise OSError(err % locals()) else: # may fail on win if the path is too long @@ -96,7 +97,7 @@ def create_dir(location): raise -def get_temp_dir(base_dir=_base_temp_dir, prefix=''): +def get_temp_dir(base_dir=_base_temp_dir, prefix=""): """ Return the path to a new existing unique temporary directory, created under the `base_dir` base directory using the `prefix` prefix. @@ -108,7 +109,7 @@ def get_temp_dir(base_dir=_base_temp_dir, prefix=''): has_base = bool(base_dir) if not has_base: - base_dir = os.getenv('SCANCODE_TMP') + base_dir = os.getenv("SCANCODE_TMP") if not base_dir: base_dir = tempfile.gettempdir() @@ -116,10 +117,11 @@ def get_temp_dir(base_dir=_base_temp_dir, prefix=''): create_dir(base_dir) if not has_base: - prefix = 'scancode-tk-' + prefix = "scancode-tk-" return tempfile.mkdtemp(prefix=prefix, dir=base_dir) + # # PATHS AND NAMES MANIPULATIONS # @@ -145,8 +147,8 @@ def is_posixpath(location): Return False if the `location` path is likely a Windows-like path using backslash as path separators (e.g. "\"). """ - has_slashes = '/' in location - has_backslashes = '\\' in location + has_slashes = "/" in location + has_backslashes = "\\" in location # windows paths with drive if location: drive, _ = ntpath.splitdrive(location) @@ -168,7 +170,7 @@ def as_posixpath(location): accepts gracefully POSIX paths on Windows. """ location = prepare_path(location) - return location.replace('\\', '/') + return location.replace("\\", "/") def as_winpath(location): @@ -177,7 +179,7 @@ def as_winpath(location): `location` path. """ location = prepare_path(location) - return location.replace('/', '\\') + return location.replace("/", "\\") def split_parent_resource(path, force_posix=False): @@ -186,7 +188,7 @@ def split_parent_resource(path, force_posix=False): """ use_posix = force_posix or is_posixpath(path) splitter = use_posix and posixpath or ntpath - path_no_trailing_speps = path.rstrip('\\/') + path_no_trailing_speps = path.rstrip("\\/") return splitter.split(path_no_trailing_speps) @@ -196,7 +198,7 @@ def resource_name(path, force_posix=False): is the last path segment. """ _left, right = split_parent_resource(path, force_posix) - return right or '' + return right or "" def file_name(path, force_posix=False): @@ -213,8 +215,8 @@ def parent_directory(path, force_posix=False, with_trail=True): """ left, _right = split_parent_resource(path, force_posix) use_posix = force_posix or is_posixpath(path) - sep = '/' if use_posix else '\\' - trail = sep if with_trail and left != sep else '' + sep = "/" if use_posix else "\\" + trail = sep if with_trail and left != sep else "" return left + trail @@ -241,19 +243,19 @@ def splitext_name(file_name, is_file=True): """ if not file_name: - return '', '' + return "", "" file_name = fsdecode(file_name) if not is_file: - return file_name, '' + return file_name, "" - if file_name.startswith('.') and '.' not in file_name[1:]: + if file_name.startswith(".") and "." not in file_name[1:]: # .dot files base name is the full name and they do not have an extension - return file_name, '' + return file_name, "" base_name, extension = posixpath.splitext(file_name) # handle composed extensions of tar.gz, bz, zx,etc - if base_name.endswith('.tar'): + if base_name.endswith(".tar"): base_name, extension2 = posixpath.splitext(base_name) extension = extension2 + extension return base_name, extension @@ -266,30 +268,36 @@ def splitext(path, force_posix=False): the file name minus its extension. Return an empty extension string for a directory. """ - base_name = '' - extension = '' + base_name = "" + extension = "" if not path: return base_name, extension - is_dir = path.endswith(('\\', '/',)) - path = as_posixpath(path).strip('/') + is_dir = path.endswith( + ( + "\\", + "/", + ) + ) + path = as_posixpath(path).strip("/") name = resource_name(path, force_posix) if is_dir: # directories never have an extension base_name = name - extension = '' - elif name.startswith('.') and '.' not in name[1:]: + extension = "" + elif name.startswith(".") and "." not in name[1:]: # .dot files base name is the full name and they do not have an extension base_name = name - extension = '' + extension = "" else: base_name, extension = posixpath.splitext(name) # handle composed extensions of tar.gz, tar.bz2, zx,etc - if base_name.endswith('.tar'): + if base_name.endswith(".tar"): base_name, extension2 = posixpath.splitext(base_name) extension = extension2 + extension return base_name, extension + # # DIRECTORY AND FILES WALKING/ITERATION # @@ -318,7 +326,7 @@ def walk(location, ignored=None, follow_symlinks=False): is_ignored = ignored(location) if ignored else False if is_ignored: if TRACE: - logger_debug('walk: ignored:', location, is_ignored) + logger_debug("walk: ignored:", location, is_ignored) return if filetype.is_file(location, follow_symlinks=follow_symlinks): @@ -331,14 +339,16 @@ def walk(location, ignored=None, follow_symlinks=False): for name in os.listdir(location): loc = os.path.join(location, name) if filetype.is_special(loc) or (ignored and ignored(loc)): - if (follow_symlinks - and filetype.is_link(loc) - and not filetype.is_broken_link(location)): + if ( + follow_symlinks + and filetype.is_link(loc) + and not filetype.is_broken_link(location) + ): pass else: if TRACE: ign = ignored and ignored(loc) - logger_debug('walk: ignored:', loc, ign) + logger_debug("walk: ignored:", loc, ign) continue # special files and symlinks are always ignored if filetype.is_dir(loc, follow_symlinks=follow_symlinks): @@ -348,7 +358,9 @@ def walk(location, ignored=None, follow_symlinks=False): yield location, dirs, files for dr in dirs: - for tripple in walk(os.path.join(location, dr), ignored, follow_symlinks=follow_symlinks): + for tripple in walk( + os.path.join(location, dr), ignored, follow_symlinks=follow_symlinks + ): yield tripple @@ -367,6 +379,8 @@ def resource_iter(location, ignored=ignore_nothing, with_dirs=True, follow_symli yield os.path.join(top, d) for f in files: yield os.path.join(top, f) + + # # COPY # @@ -450,7 +464,7 @@ def copytime(src, dst): """ errors = [] st = os.stat(src) - if hasattr(os, 'utime'): + if hasattr(os, "utime"): try: os.utime(dst, (st.st_atime, st.st_mtime)) except OSError as why: @@ -461,6 +475,7 @@ def copytime(src, dst): errors.append((src, dst, str(why))) return errors + # # PERMISSIONS # @@ -516,6 +531,7 @@ def chmod_tree(location, flags): for f in files: chmod(os.path.join(top, f), flags, recurse=False) + # # DELETION # @@ -527,7 +543,7 @@ def _rm_handler(function, path, excinfo): # NOQA This retries deleting once before giving up. """ if TRACE: - logger_debug('_rm_handler:', 'path:', path, 'excinfo:', excinfo) + logger_debug("_rm_handler:", "path:", path, "excinfo:", excinfo) if function in (os.rmdir, os.listdir): try: chmod(path, RW, recurse=True) @@ -536,7 +552,7 @@ def _rm_handler(function, path, excinfo): # NOQA pass if os.path.exists(path): - logger.warning('Failed to delete directory %s', path) + logger.warning("Failed to delete directory %s", path) elif function == os.remove: try: @@ -545,7 +561,7 @@ def _rm_handler(function, path, excinfo): # NOQA pass if os.path.exists(path): - logger.warning('Failed to delete file %s', path) + logger.warning("Failed to delete file %s", path) def delete(location, _err_handler=_rm_handler): diff --git a/src/commoncode/functional.py b/src/commoncode/functional.py index 8bb6f64f..eaa7376d 100644 --- a/src/commoncode/functional.py +++ b/src/commoncode/functional.py @@ -7,9 +7,8 @@ # import functools - -from types import GeneratorType from array import array +from types import GeneratorType def flatten(seq): @@ -87,8 +86,7 @@ def memoized(*args, **kwargs): if kwargs: return fun(*args, **kwargs) # convert any list args to a tuple - args = tuple(tuple(arg) if isinstance(arg, (list, tuple, array)) else arg - for arg in args) + args = tuple(tuple(arg) if isinstance(arg, (list, tuple, array)) else arg for arg in args) try: return memos[args] except KeyError: diff --git a/src/commoncode/hash.py b/src/commoncode/hash.py index 61be3758..954d3f4b 100644 --- a/src/commoncode/hash.py +++ b/src/commoncode/hash.py @@ -11,9 +11,9 @@ import sys from functools import partial +from commoncode import filetype from commoncode.codec import bin_to_num from commoncode.codec import urlsafe_b64encode -from commoncode import filetype """ Hashes and checksums. @@ -33,19 +33,18 @@ def _hash_mod(bitsize, hmodule): """ class hasher(object): - def __init__(self, msg=None): self.digest_size = bitsize // 8 - self.h = msg and hmodule(msg).digest()[:self.digest_size] or None + self.h = msg and hmodule(msg).digest()[: self.digest_size] or None def digest(self): return bytes(self.h) def hexdigest(self): - return self.h and binascii.hexlify(self.h).decode('utf-8') + return self.h and binascii.hexlify(self.h).decode("utf-8") def b64digest(self): - return self.h and urlsafe_b64encode(self.h).decode('utf-8') + return self.h and urlsafe_b64encode(self.h).decode("utf-8") def intdigest(self): return self.h and int(bin_to_num(self.h)) @@ -72,7 +71,7 @@ def intdigest(self): 160: _hash_mod(160, hashlib.sha1), 256: _hash_mod(256, hashlib.sha256), 384: _hash_mod(384, hashlib.sha384), - 512: _hash_mod(512, hashlib.sha512) + 512: _hash_mod(512, hashlib.sha512), } @@ -94,29 +93,29 @@ def __init__(self, msg=None): def _compute(self, msg): # note: bytes interpolation is new in Python 3.5 - git_blob_msg = b'blob %d\0%s' % (len(msg), msg) + git_blob_msg = b"blob %d\0%s" % (len(msg), msg) return hashlib.sha1(git_blob_msg).digest() def digest(self): return bytes(self.h) def hexdigest(self): - return self.h and binascii.hexlify(self.h).decode('utf-8') + return self.h and binascii.hexlify(self.h).decode("utf-8") def b64digest(self): - return self.h and urlsafe_b64encode(self.h).decode('utf-8') + return self.h and urlsafe_b64encode(self.h).decode("utf-8") def intdigest(self): return self.h and int(bin_to_num(self.h)) _hashmodules_by_name = { - 'md5': get_hasher(128), - 'sha1': get_hasher(160), - 'sha1_git': sha1_git_hasher, - 'sha256': get_hasher(256), - 'sha384': get_hasher(384), - 'sha512': get_hasher(512) + "md5": get_hasher(128), + "sha1": get_hasher(160), + "sha1_git": sha1_git_hasher, + "sha256": get_hasher(256), + "sha384": get_hasher(384), + "sha512": get_hasher(512), } @@ -131,7 +130,7 @@ def checksum(location, name, base64=False): hasher = _hashmodules_by_name[name] # fixme: we should read in chunks? - with open(location, 'rb') as f: + with open(location, "rb") as f: hashable = f.read() hashed = hasher(hashable) @@ -142,30 +141,30 @@ def checksum(location, name, base64=False): def md5(location): - return checksum(location, name='md5', base64=False) + return checksum(location, name="md5", base64=False) def sha1(location): - return checksum(location, name='sha1', base64=False) + return checksum(location, name="sha1", base64=False) def b64sha1(location): - return checksum(location, name='sha1', base64=True) + return checksum(location, name="sha1", base64=True) def sha256(location): - return checksum(location, name='sha256', base64=False) + return checksum(location, name="sha256", base64=False) def sha512(location): - return checksum(location, name='sha512', base64=False) + return checksum(location, name="sha512", base64=False) def sha1_git(location): - return checksum(location, name='sha1_git', base64=False) + return checksum(location, name="sha1_git", base64=False) -def multi_checksums(location, checksum_names=('md5', 'sha1', 'sha256', 'sha512', 'sha1_git')): +def multi_checksums(location, checksum_names=("md5", "sha1", "sha256", "sha512", "sha1_git")): """ Return a mapping of hexdigest checksums keyed by checksum name from the content of the file at `location`. Use the `checksum_names` list of checksum names. @@ -177,7 +176,7 @@ def multi_checksums(location, checksum_names=('md5', 'sha1', 'sha256', 'sha512', return results # fixme: we should read in chunks? - with open(location, 'rb') as f: + with open(location, "rb") as f: hashable = f.read() for name in checksum_names: diff --git a/src/commoncode/ignore.py b/src/commoncode/ignore.py index dc21d678..c19ccdc3 100644 --- a/src/commoncode/ignore.py +++ b/src/commoncode/ignore.py @@ -38,10 +38,7 @@ def is_ignore_file(location): """ Return True if the location is an ignore file. """ - return ( - filetype.is_file(location) - and fileutils.file_name(location) == '.scancodeignore' - ) + return filetype.is_file(location) and fileutils.file_name(location) == ".scancodeignore" def get_ignores(location, include_defaults=True): @@ -59,244 +56,236 @@ def get_ignores(location, include_defaults=True): unignores.update(uni) return ignores, unignores + # # Default ignores # ignores_MacOSX = { - '.DS_Store': 'Default ignore: MacOSX artifact', - '._.DS_Store': 'Default ignore: MacOSX artifact', - '__MACOSX': 'Default ignore: MacOSX artifact', - '.AppleDouble': 'Default ignore: MacOSX artifact', - '.LSOverride': 'Default ignore: MacOSX artifact', - '.DocumentRevisions-V100': 'Default ignore: MacOSX artifact', - '.fseventsd': 'Default ignore: MacOSX artifact', - '.Spotlight-V100': 'Default ignore: MacOSX artifact', - '.VolumeIcon.icns': 'Default ignore: MacOSX artifact', - - '.journal': 'Default ignore: MacOSX DMG/HFS+ artifact', - '.journal_info_block': 'Default ignore: MacOSX DMG/HFS+ artifact', - '.Trashes': 'Default ignore: MacOSX DMG/HFS+ artifact', - r'\[HFS+ Private Data\]': 'Default ignore: MacOSX DMG/HFS+ artifact private data', + ".DS_Store": "Default ignore: MacOSX artifact", + "._.DS_Store": "Default ignore: MacOSX artifact", + "__MACOSX": "Default ignore: MacOSX artifact", + ".AppleDouble": "Default ignore: MacOSX artifact", + ".LSOverride": "Default ignore: MacOSX artifact", + ".DocumentRevisions-V100": "Default ignore: MacOSX artifact", + ".fseventsd": "Default ignore: MacOSX artifact", + ".Spotlight-V100": "Default ignore: MacOSX artifact", + ".VolumeIcon.icns": "Default ignore: MacOSX artifact", + ".journal": "Default ignore: MacOSX DMG/HFS+ artifact", + ".journal_info_block": "Default ignore: MacOSX DMG/HFS+ artifact", + ".Trashes": "Default ignore: MacOSX DMG/HFS+ artifact", + r"\[HFS+ Private Data\]": "Default ignore: MacOSX DMG/HFS+ artifact private data", } ignores_Windows = { - 'Thumbs.db': 'Default ignore: Windows artifact', - 'ehthumbs.db': 'Default ignore: Windows artifact', - 'Desktop.ini': 'Default ignore: Windows artifact', - '$RECYCLE.BIN': 'Default ignore: Windows artifact', - '*.lnk': 'Default ignore: Windows artifact', - 'System Volume Information': 'Default ignore: Windows FS artifact', - 'NTUSER.DAT*': 'Default ignore: Windows FS artifact', + "Thumbs.db": "Default ignore: Windows artifact", + "ehthumbs.db": "Default ignore: Windows artifact", + "Desktop.ini": "Default ignore: Windows artifact", + "$RECYCLE.BIN": "Default ignore: Windows artifact", + "*.lnk": "Default ignore: Windows artifact", + "System Volume Information": "Default ignore: Windows FS artifact", + "NTUSER.DAT*": "Default ignore: Windows FS artifact", } ignores_Linux = { - '.directory': 'Default ignore: KDE artifact', - '.Trash-*': 'Default ignore: Linux/Gome/KDE artifact', + ".directory": "Default ignore: KDE artifact", + ".Trash-*": "Default ignore: Linux/Gome/KDE artifact", } ignores_IDEs = { - '*.el': 'Default ignore: EMACS Elisp artifact', - '*.swp': 'Default ignore: VIM artifact', - '.project': 'Default ignore: Eclipse IDE artifact', - '.pydevproject': 'Default ignore: Eclipse IDE artifact', - '.settings': 'Default ignore: Eclipse IDE artifact', - '.eclipse': 'Default ignore: Eclipse IDE artifact', - '.loadpath': 'Default ignore: Eclipse IDE artifact', - '*.launch': 'Default ignore: Eclipse IDE artifact', - '.cproject': 'Default ignore: Eclipse IDE artifact', - '.cdtproject': 'Default ignore: Eclipse IDE artifact', - '.classpath': 'Default ignore: Eclipse IDE artifact', - '.buildpath': 'Default ignore: Eclipse IDE artifact', - '.texlipse': 'Default ignore: Eclipse IDE artifact', - - '*.iml': 'Default ignore: JetBrains IDE artifact', - '*.ipr': 'Default ignore: JetBrains IDE artifact', - '*.iws': 'Default ignore: JetBrains IDE artifact', - '.idea/': 'Default ignore: JetBrains IDE artifact', - '.idea_modules/': 'Default ignore: JetBrains IDE artifact', - - '*.kdev4': 'Default ignore: Kdevelop artifact', - '.kdev4/': 'Default ignore: Kdevelop artifact', - - '*.nib': 'Default ignore: Apple Xcode artifact', - '*.plst': 'Default ignore: Apple Xcode plist artifact', - '*.pbxuser': 'Default ignore: Apple Xcode artifact', - '*.pbxproj': 'Default ignore: Apple Xcode artifact', - 'xcuserdata': 'Default ignore: Apple Xcode artifact', - '*.xcuserstate': 'Default ignore: Apple Xcode artifact', - - '*.csproj': 'Default ignore: Microsoft VS project artifact', - '*.unityproj': 'Default ignore: Microsoft VS project artifact', - '*.sln': 'Default ignore: Microsoft VS project artifact', - '*.sluo': 'Default ignore: Microsoft VS project artifact', - '*.suo': 'Default ignore: Microsoft VS project artifact', - '*.user': 'Default ignore: Microsoft VS project artifact', - '*.sln.docstates': 'Default ignore: Microsoft VS project artifact', - '*.dsw': 'Default ignore: Microsoft VS project artifact', - - '.editorconfig': 'Default ignore: Editor config artifact', - - ' Leiningen.gitignore': 'Default ignore: Leiningen artifact', - '.architect': 'Default ignore: ExtJS artifact', - '*.tmproj': 'Default ignore: Textmate artifact', - '*.tmproject': 'Default ignore: Textmate artifact', + "*.el": "Default ignore: EMACS Elisp artifact", + "*.swp": "Default ignore: VIM artifact", + ".project": "Default ignore: Eclipse IDE artifact", + ".pydevproject": "Default ignore: Eclipse IDE artifact", + ".settings": "Default ignore: Eclipse IDE artifact", + ".eclipse": "Default ignore: Eclipse IDE artifact", + ".loadpath": "Default ignore: Eclipse IDE artifact", + "*.launch": "Default ignore: Eclipse IDE artifact", + ".cproject": "Default ignore: Eclipse IDE artifact", + ".cdtproject": "Default ignore: Eclipse IDE artifact", + ".classpath": "Default ignore: Eclipse IDE artifact", + ".buildpath": "Default ignore: Eclipse IDE artifact", + ".texlipse": "Default ignore: Eclipse IDE artifact", + "*.iml": "Default ignore: JetBrains IDE artifact", + "*.ipr": "Default ignore: JetBrains IDE artifact", + "*.iws": "Default ignore: JetBrains IDE artifact", + ".idea/": "Default ignore: JetBrains IDE artifact", + ".idea_modules/": "Default ignore: JetBrains IDE artifact", + "*.kdev4": "Default ignore: Kdevelop artifact", + ".kdev4/": "Default ignore: Kdevelop artifact", + "*.nib": "Default ignore: Apple Xcode artifact", + "*.plst": "Default ignore: Apple Xcode plist artifact", + "*.pbxuser": "Default ignore: Apple Xcode artifact", + "*.pbxproj": "Default ignore: Apple Xcode artifact", + "xcuserdata": "Default ignore: Apple Xcode artifact", + "*.xcuserstate": "Default ignore: Apple Xcode artifact", + "*.csproj": "Default ignore: Microsoft VS project artifact", + "*.unityproj": "Default ignore: Microsoft VS project artifact", + "*.sln": "Default ignore: Microsoft VS project artifact", + "*.sluo": "Default ignore: Microsoft VS project artifact", + "*.suo": "Default ignore: Microsoft VS project artifact", + "*.user": "Default ignore: Microsoft VS project artifact", + "*.sln.docstates": "Default ignore: Microsoft VS project artifact", + "*.dsw": "Default ignore: Microsoft VS project artifact", + ".editorconfig": "Default ignore: Editor config artifact", + " Leiningen.gitignore": "Default ignore: Leiningen artifact", + ".architect": "Default ignore: ExtJS artifact", + "*.tmproj": "Default ignore: Textmate artifact", + "*.tmproject": "Default ignore: Textmate artifact", } ignores_web = { - '.htaccess': 'Default ignore: .htaccess file', - 'robots.txt': 'Default ignore: robots file', - 'humans.txt': 'Default ignore: robots file', - 'web.config': 'Default ignore: web config', - '.htaccess.sample': 'Default ignore: .htaccess file', + ".htaccess": "Default ignore: .htaccess file", + "robots.txt": "Default ignore: robots file", + "humans.txt": "Default ignore: robots file", + "web.config": "Default ignore: web config", + ".htaccess.sample": "Default ignore: .htaccess file", } ignores_Maven = { - 'pom.xml.tag': 'Default ignore: Maven artifact', - 'pom.xml.releaseBackup': 'Default ignore: Maven artifact', - 'pom.xml.versionsBackup': 'Default ignore: Maven artifact', - 'pom.xml.next': 'Default ignore: Maven artifact', - 'release.properties': 'Default ignore: Maven artifact', - 'dependency-reduced-pom.xml': 'Default ignore: Maven artifact', - 'buildNumber.properties': 'Default ignore: Maven artifact', + "pom.xml.tag": "Default ignore: Maven artifact", + "pom.xml.releaseBackup": "Default ignore: Maven artifact", + "pom.xml.versionsBackup": "Default ignore: Maven artifact", + "pom.xml.next": "Default ignore: Maven artifact", + "release.properties": "Default ignore: Maven artifact", + "dependency-reduced-pom.xml": "Default ignore: Maven artifact", + "buildNumber.properties": "Default ignore: Maven artifact", } ignores_VCS = { - '.bzr': 'Default ignore: Bazaar artifact', - '.bzrignore': 'Default ignore: Bazaar config artifact', - - '.git': 'Default ignore: Git artifact', - '.gitignore': 'Default ignore: Git config artifact', - '.gitattributes': 'Default ignore: Git config artifact', - - '.hg': 'Default ignore: Mercurial artifact', - '.hgignore': 'Default ignore: Mercurial config artifact', - - '.repo': 'Default ignore: Multiple Git repository artifact', - - '.svn': 'Default ignore: SVN artifact', - '.svnignore': 'Default ignore: SVN config artifact', - - '.tfignore': 'Default ignore: Microsft TFS config artifact', - - 'vssver.scc': 'Default ignore: Visual Source Safe artifact', - - 'CVS': 'Default ignore: CVS artifact', - '.cvsignore': 'Default ignore: CVS config artifact', - - '*/_MTN': 'Default ignore: Monotone artifact', - '*/_darcs': 'Default ignore: Darcs artifact', - '*/{arch}': 'Default ignore: GNU Arch artifact', + ".bzr": "Default ignore: Bazaar artifact", + ".bzrignore": "Default ignore: Bazaar config artifact", + ".git": "Default ignore: Git artifact", + ".gitignore": "Default ignore: Git config artifact", + ".gitattributes": "Default ignore: Git config artifact", + ".hg": "Default ignore: Mercurial artifact", + ".hgignore": "Default ignore: Mercurial config artifact", + ".repo": "Default ignore: Multiple Git repository artifact", + ".svn": "Default ignore: SVN artifact", + ".svnignore": "Default ignore: SVN config artifact", + ".tfignore": "Default ignore: Microsft TFS config artifact", + "vssver.scc": "Default ignore: Visual Source Safe artifact", + "CVS": "Default ignore: CVS artifact", + ".cvsignore": "Default ignore: CVS config artifact", + "*/_MTN": "Default ignore: Monotone artifact", + "*/_darcs": "Default ignore: Darcs artifact", + "*/{arch}": "Default ignore: GNU Arch artifact", } ignores_Medias = { - 'pspbrwse.jbf': 'Default ignore: Paintshop browse file', - 'Thumbs.db': 'Default ignore: Image thumbnails DB', - 'Thumbs.db:encryptable': 'Default ignore: Image thumbnails DB', - 'thumbs/': 'Default ignore: Image thumbnails DB', - '_thumbs/': 'Default ignore: Image thumbnails DB', + "pspbrwse.jbf": "Default ignore: Paintshop browse file", + "Thumbs.db": "Default ignore: Image thumbnails DB", + "Thumbs.db:encryptable": "Default ignore: Image thumbnails DB", + "thumbs/": "Default ignore: Image thumbnails DB", + "_thumbs/": "Default ignore: Image thumbnails DB", } ignores_Build_scripts = { - 'Makefile.in': 'Default ignore: automake artifact', - 'Makefile.am': 'Default ignore: automake artifact', - 'autom4te.cache': 'Default ignore: autoconf artifact', - '*.m4': 'Default ignore: autotools artifact', - 'configure': 'Default ignore: Configure script', - 'configure.bat': 'Default ignore: Configure script', - 'configure.sh': 'Default ignore: Configure script', - 'configure.ac': 'Default ignore: Configure script', - 'config.guess': 'Default ignore: Configure script', - 'config.sub': 'Default ignore: Configure script', - 'compile': 'Default ignore: autoconf artifact', - 'depcomp': 'Default ignore: autoconf artifact', - 'ltmain.sh': 'Default ignore: libtool autoconf artifact', - 'install-sh': 'Default ignore: autoconf artifact', - 'missing': 'Default ignore: autoconf artifact', - 'mkinstalldirs': 'Default ignore: autoconf artifact', - 'stamp-h1': 'Default ignore: autoconf artifact', - 'm4/': 'Default ignore: autoconf artifact', - 'autogen.sh': 'Default ignore: autotools artifact', - 'autogen.sh': 'Default ignore: autotools artifact', - - 'CMakeCache.txt': 'Default ignore: CMake artifact', - 'cmake_install.cmake': 'Default ignore: CMake artifact', - 'install_manifest.txt': 'Default ignore: CMake artifact', + "Makefile.in": "Default ignore: automake artifact", + "Makefile.am": "Default ignore: automake artifact", + "autom4te.cache": "Default ignore: autoconf artifact", + "*.m4": "Default ignore: autotools artifact", + "configure": "Default ignore: Configure script", + "configure.bat": "Default ignore: Configure script", + "configure.sh": "Default ignore: Configure script", + "configure.ac": "Default ignore: Configure script", + "config.guess": "Default ignore: Configure script", + "config.sub": "Default ignore: Configure script", + "compile": "Default ignore: autoconf artifact", + "depcomp": "Default ignore: autoconf artifact", + "ltmain.sh": "Default ignore: libtool autoconf artifact", + "install-sh": "Default ignore: autoconf artifact", + "missing": "Default ignore: autoconf artifact", + "mkinstalldirs": "Default ignore: autoconf artifact", + "stamp-h1": "Default ignore: autoconf artifact", + "m4/": "Default ignore: autoconf artifact", + "autogen.sh": "Default ignore: autotools artifact", + "autogen.sh": "Default ignore: autotools artifact", + "CMakeCache.txt": "Default ignore: CMake artifact", + "cmake_install.cmake": "Default ignore: CMake artifact", + "install_manifest.txt": "Default ignore: CMake artifact", } ignores_CI = { - '.travis.yml': 'Default ignore: Travis config', - '.coveragerc': 'Default ignore: Coverall config', + ".travis.yml": "Default ignore: Travis config", + ".coveragerc": "Default ignore: Coverall config", } ignores_Python = { - 'pip-selfcheck.json': 'Default ignore: Pip workfile', - 'pytest.ini': 'Default ignore: Python pytest config', - 'tox.ini': 'Default ignore: Python tox config', - '__pycache__/': 'Default ignore: Python bytecode cache', - '.installed.cfg': 'Default ignore: Python Buildout artifact', - 'pip-log.txt': 'Default ignore: Python pip artifact', - 'pip-delete-this-directory.txt': 'Default ignore: Python pip artifact', - 'pyvenv.cfg': 'Default ignore: Python virtualenv artifact', + "pip-selfcheck.json": "Default ignore: Pip workfile", + "pytest.ini": "Default ignore: Python pytest config", + "tox.ini": "Default ignore: Python tox config", + "__pycache__/": "Default ignore: Python bytecode cache", + ".installed.cfg": "Default ignore: Python Buildout artifact", + "pip-log.txt": "Default ignore: Python pip artifact", + "pip-delete-this-directory.txt": "Default ignore: Python pip artifact", + "pyvenv.cfg": "Default ignore: Python virtualenv artifact", } ignores_I18N = { - '*.mo': 'Default ignore: Translation file', - '*.pot': 'Default ignore: Translation file', - '.localized': 'Default ignore: localized file', + "*.mo": "Default ignore: Translation file", + "*.pot": "Default ignore: Translation file", + ".localized": "Default ignore: localized file", } ignores_coverage_and_tests = { - '*.gcno': 'Default ignore: GCC coverage', - '*.gcda': 'Default ignore: GCC coverage', - '*.gcov': 'Default ignore: GCC coverage', - '.last_cover_stats': 'Default ignore: Perl coverage', - 'htmlcov/': 'Default ignore: Python coverage', - '.tox/': 'Default ignore: Tox tem dir', - '.coverage': 'Default ignore: Python coverage', - '.coverage.*': 'Default ignore: Python coverage', - 'nosetests.xml': 'Default ignore: Python nose tests', - 'coverage.xml': 'Default ignore: Python coverage', - '/spec/reports/': 'Default ignore: Ruby Rails test report', - '/rdoc/': 'Default ignore: Ruby doc', - '.rvmrc': 'Default ignore: Ruby RVM', - '.sass-cache': 'Default ignore: Saas cache', - '*.css.map': 'Default ignore: Saas map', - 'phpunit.xml': 'Default ignore: phpunit', - '*.VisualState.xml': 'Default ignore: Nunit', - 'TestResult.xml': 'Default ignore: Nunit', + "*.gcno": "Default ignore: GCC coverage", + "*.gcda": "Default ignore: GCC coverage", + "*.gcov": "Default ignore: GCC coverage", + ".last_cover_stats": "Default ignore: Perl coverage", + "htmlcov/": "Default ignore: Python coverage", + ".tox/": "Default ignore: Tox tem dir", + ".coverage": "Default ignore: Python coverage", + ".coverage.*": "Default ignore: Python coverage", + "nosetests.xml": "Default ignore: Python nose tests", + "coverage.xml": "Default ignore: Python coverage", + "/spec/reports/": "Default ignore: Ruby Rails test report", + "/rdoc/": "Default ignore: Ruby doc", + ".rvmrc": "Default ignore: Ruby RVM", + ".sass-cache": "Default ignore: Saas cache", + "*.css.map": "Default ignore: Saas map", + "phpunit.xml": "Default ignore: phpunit", + "*.VisualState.xml": "Default ignore: Nunit", + "TestResult.xml": "Default ignore: Nunit", } ignores_Misc = { - 'pax_global_header': 'Default ignore: Pax header file', - 'C++.gitignore': 'Default ignore: C++.gitignore', - '.gwt/': 'Default ignore: GWT compilation logs', - '.gwt-tmp/': 'Default ignore: GWT temp files', - 'gradle-app.setting': 'Default ignore: Graddle app settings', - 'hs_err_pid*': 'Default ignore: Java VM crash logs', - '.grunt': 'Default ignore: Grunt intermediate storage', - '.history': 'Default ignore: History file', - '.~lock.*#': 'Default ignore: LibreOffice locks', - '/.ssh': 'Default ignore: SSH configuration', + "pax_global_header": "Default ignore: Pax header file", + "C++.gitignore": "Default ignore: C++.gitignore", + ".gwt/": "Default ignore: GWT compilation logs", + ".gwt-tmp/": "Default ignore: GWT temp files", + "gradle-app.setting": "Default ignore: Graddle app settings", + "hs_err_pid*": "Default ignore: Java VM crash logs", + ".grunt": "Default ignore: Grunt intermediate storage", + ".history": "Default ignore: History file", + ".~lock.*#": "Default ignore: LibreOffice locks", + "/.ssh": "Default ignore: SSH configuration", } default_ignores = {} -default_ignores.update(chain(*[d.items() for d in [ - ignores_MacOSX, - ignores_Windows, - ignores_Linux, - ignores_IDEs, - ignores_web, - ignores_Maven, - ignores_VCS, - ignores_Medias, - ignores_Build_scripts, - ignores_CI, - ignores_Python, - ignores_I18N, - ignores_coverage_and_tests, - ignores_Misc, - ignores_Build_scripts, -]])) +default_ignores.update( + chain( + *[ + d.items() + for d in [ + ignores_MacOSX, + ignores_Windows, + ignores_Linux, + ignores_IDEs, + ignores_web, + ignores_Maven, + ignores_VCS, + ignores_Medias, + ignores_Build_scripts, + ignores_CI, + ignores_Python, + ignores_I18N, + ignores_coverage_and_tests, + ignores_Misc, + ignores_Build_scripts, + ] + ] + ) +) diff --git a/src/commoncode/paths.py b/src/commoncode/paths.py index 8718c503..f9d11d66 100644 --- a/src/commoncode/paths.py +++ b/src/commoncode/paths.py @@ -9,14 +9,13 @@ import ntpath import posixpath import re - from os.path import commonprefix -from commoncode.text import as_unicode -from commoncode.text import toascii from commoncode.fileutils import as_posixpath from commoncode.fileutils import as_winpath from commoncode.fileutils import is_posixpath +from commoncode.text import as_unicode +from commoncode.text import toascii """ Various path utilities such as common prefix and suffix functions, conversion @@ -53,18 +52,15 @@ def safe_path(path, posix=False, preserve_spaces=False, posix_only=False): segments = [s.strip() for s in path.split(path_sep) if s.strip()] segments = [ - portable_filename( - s, - preserve_spaces=preserve_spaces, - posix_only=posix_only - ) for s in segments + portable_filename(s, preserve_spaces=preserve_spaces, posix_only=posix_only) + for s in segments ] if not segments: - return '_' + return "_" # always return posix - path = '/'.join(segments) + path = "/".join(segments) return as_posixpath(path) @@ -78,7 +74,7 @@ def path_handlers(path, posix=True): is_posix = is_posixpath(path) use_posix = posix or is_posix pathmod = use_posix and posixpath or ntpath - path_sep = '/' if use_posix else '\\' + path_sep = "/" if use_posix else "\\" return pathmod, path_sep @@ -94,11 +90,11 @@ def resolve(path, posix=True): Windows path with blackslash separators otherwise. """ if not path: - return '.' + return "." path = path.strip() if not path: - return '.' + return "." if not is_posixpath(path): path = as_winpath(path) @@ -110,7 +106,7 @@ def resolve(path, posix=True): segments = [s.strip() for s in path.split(path_sep) if s.strip()] # remove empty (// or ///) or blank (space only) or single dot segments - segments = [s for s in segments if s and s != '.'] + segments = [s for s in segments if s and s != "."] path = path_sep.join(segments) @@ -123,30 +119,30 @@ def resolve(path, posix=True): segments = [s.strip() for s in segments if s and s.strip()] # is this a windows absolute path? if yes strip the colon to make this relative - if segments and len(segments[0]) == 2 and segments[0].endswith(':'): + if segments and len(segments[0]) == 2 and segments[0].endswith(":"): segments[0] = segments[0][:-1] # replace any remaining (usually leading) .. segment with a literal "dotdot" - dotdot = 'dotdot' - dd = '..' + dotdot = "dotdot" + dd = ".." segments = [dotdot if s == dd else s for s in segments if s] if segments: path = path_sep.join(segments) else: - path = '.' + path = "." path = as_posixpath(path) return path -legal_punctuation = r'!\#$%&\(\)\+,\-\.;\=@\[\]_\{\}\~' -legal_spaces = r' ' -legal_alphanumeric = r'A-Za-z0-9' +legal_punctuation = r"!\#$%&\(\)\+,\-\.;\=@\[\]_\{\}\~" +legal_spaces = r" " +legal_alphanumeric = r"A-Za-z0-9" legal_chars = legal_alphanumeric + legal_punctuation legal_chars_inc_spaces = legal_chars + legal_spaces -illegal_chars_re = r'[^' + legal_chars + r']' -illegal_chars_exc_spaces_re = r'[^' + legal_chars_inc_spaces + r']' +illegal_chars_re = r"[^" + legal_chars + r"]" +illegal_chars_exc_spaces_re = r"[^" + legal_chars_inc_spaces + r"]" replace_illegal_chars = re.compile(illegal_chars_re).sub replace_illegal_chars_exc_spaces = re.compile(illegal_chars_exc_spaces_re).sub @@ -154,18 +150,38 @@ def resolve(path, posix=True): posix_legal_punctuation = r'<:"/>\|\*\^\\\'`\?' + legal_punctuation posix_legal_chars = legal_alphanumeric + posix_legal_punctuation posix_legal_chars_inc_spaces = posix_legal_chars + legal_spaces -posix_illegal_chars_re = r'[^' + posix_legal_chars + r']' -posix_illegal_chars_exc_spaces_re = r'[^' + posix_legal_chars_inc_spaces + r']' +posix_illegal_chars_re = r"[^" + posix_legal_chars + r"]" +posix_illegal_chars_exc_spaces_re = r"[^" + posix_legal_chars_inc_spaces + r"]" replace_illegal_posix_chars = re.compile(posix_illegal_chars_re).sub -replace_illegal_posix_chars_exc_spaces = re.compile( - posix_illegal_chars_exc_spaces_re).sub - - -ILLEGAL_WINDOWS_NAMES = set([ - 'com1', 'com2', 'com3', 'com4', 'com5', 'com6', 'com7', 'com8', 'com9', - 'lpt1', 'lpt2', 'lpt3', 'lpt4', 'lpt5', 'lpt6', 'lpt7', 'lpt8', 'lpt9', - 'aux', 'con', 'nul', 'prn' -]) +replace_illegal_posix_chars_exc_spaces = re.compile(posix_illegal_chars_exc_spaces_re).sub + + +ILLEGAL_WINDOWS_NAMES = set( + [ + "com1", + "com2", + "com3", + "com4", + "com5", + "com6", + "com7", + "com8", + "com9", + "lpt1", + "lpt2", + "lpt3", + "lpt4", + "lpt5", + "lpt6", + "lpt7", + "lpt8", + "lpt9", + "aux", + "con", + "nul", + "prn", + ] +) def portable_filename(filename, preserve_spaces=False, posix_only=False): @@ -192,32 +208,32 @@ def portable_filename(filename, preserve_spaces=False, posix_only=False): filename = toascii(filename, translit=True) if not filename: - return '_' + return "_" if posix_only: if preserve_spaces: - filename = replace_illegal_posix_chars_exc_spaces('_', filename) + filename = replace_illegal_posix_chars_exc_spaces("_", filename) else: - filename = replace_illegal_posix_chars('_', filename) + filename = replace_illegal_posix_chars("_", filename) else: if preserve_spaces: - filename = replace_illegal_chars_exc_spaces('_', filename) + filename = replace_illegal_chars_exc_spaces("_", filename) else: - filename = replace_illegal_chars('_', filename) + filename = replace_illegal_chars("_", filename) if not posix_only: - basename, dot, extension = filename.partition('.') + basename, dot, extension = filename.partition(".") if basename.lower() in ILLEGAL_WINDOWS_NAMES: - filename = ''.join([basename, '_', dot, extension]) + filename = "".join([basename, "_", dot, extension]) # no name made only of dots. - if set(filename) == set(['.']): - filename = 'dot' * len(filename) + if set(filename) == set(["."]): + filename = "dot" * len(filename) # replaced any leading dotdot - if filename != '..' and filename.startswith('..'): - while filename.startswith('..'): - filename = filename.replace('..', '__', 1) + if filename != ".." and filename.startswith(".."): + while filename.startswith(".."): + filename = filename.replace("..", "__", 1) return filename @@ -233,7 +249,12 @@ def common_prefix(s1, s2): """ if not s1 or not s2: return None, 0 - common = commonprefix((s1, s2,)) + common = commonprefix( + ( + s1, + s2, + ) + ) if common: return common, len(common) else: @@ -276,8 +297,8 @@ def split(p): """ if not p: return [] - p = p.strip('/').split('/') - return [] if p == [''] else p + p = p.strip("/").split("/") + return [] if p == [""] else p def _common_path(p1, p2, common_func): @@ -287,5 +308,5 @@ def _common_path(p1, p2, common_func): function. """ common, lgth = common_func(split(p1), split(p2)) - common = '/'.join(common) if common else None + common = "/".join(common) if common else None return common, lgth diff --git a/src/commoncode/resource.py b/src/commoncode/resource.py index 2e799481..6ce1536a 100644 --- a/src/commoncode/resource.py +++ b/src/commoncode/resource.py @@ -21,9 +21,9 @@ from os.path import isfile from os.path import join from os.path import normpath +from posixpath import dirname as posixpath_parent from posixpath import join as posixpath_join from posixpath import normpath as posixpath_normpath -from posixpath import dirname as posixpath_parent import attr @@ -33,7 +33,7 @@ # alway have something there. import tempfile - temp_dir = tempfile.mkdtemp(prefix='scancode-resource-cache') + temp_dir = tempfile.mkdtemp(prefix="scancode-resource-cache") from commoncode import ignore from commoncode.datautils import List @@ -77,7 +77,7 @@ def logger_debug(*args): logger.setLevel(logging.DEBUG) def logger_debug(*args): - return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args)) + return logger.debug(" ".join(isinstance(a, str) and a or repr(a) for a in args)) class ResourceNotInCache(Exception): @@ -98,11 +98,11 @@ def skip_ignored(location): if TRACE_DEEP: logger_debug() logger_debug( - 'Codebase.populate: walk: ignored loc:', + "Codebase.populate: walk: ignored loc:", location, - 'ignored:', + "ignored:", ignored(location), - 'is_special:', + "is_special:", is_special(location), ) @@ -157,20 +157,18 @@ class Header(object): should create a Header and append it to the Codebase.headers list. """ - tool_name = String(help='Name of the tool used such as scancode-toolkit.') - tool_version = String(default='', help='Tool version used such as v1.2.3.') - options = Mapping( - help='Mapping of key/values describing the options used with this tool.') - notice = String(default='', help='Notice text for this tool.') - start_timestamp = String(help='Start timestamp for this header.') - end_timestamp = String(help='End timestamp for this header.') - output_format_version = String( - help='Version for the output data format, such as v1.1 .') - duration = String(help='Scan duration in seconds.') - message = String(help='Message text.') - errors = List(help='List of error messages.') - warnings = List(help='List of warning messages.') - extra_data = Mapping(help='Mapping of extra key/values for this tool.') + tool_name = String(help="Name of the tool used such as scancode-toolkit.") + tool_version = String(default="", help="Tool version used such as v1.2.3.") + options = Mapping(help="Mapping of key/values describing the options used with this tool.") + notice = String(default="", help="Notice text for this tool.") + start_timestamp = String(help="Start timestamp for this header.") + end_timestamp = String(help="End timestamp for this header.") + output_format_version = String(help="Version for the output data format, such as v1.1 .") + duration = String(help="Scan duration in seconds.") + message = String(help="Message text.") + errors = List(help="List of error messages.") + warnings = List(help="List of warning messages.") + extra_data = Mapping(help="Mapping of extra key/values for this tool.") def to_dict(self): return attr.asdict(self, dict_factory=dict) @@ -204,28 +202,28 @@ class Codebase: # we do not really need slots but this is a way to ensure we have tight # control on object attributes __slots__ = ( - 'max_depth', - 'location', - 'has_single_resource', - 'resource_attributes', - 'resource_class', - 'root', - 'is_file', - 'temp_dir', - 'resources_by_path', - 'resources_count', - 'paths', - 'max_in_memory', - 'all_in_memory', - 'all_on_disk', - 'cache_dir', - 'headers', - 'current_header', - 'codebase_attributes', - 'attributes', - 'counters', - 'timings', - 'errors', + "max_depth", + "location", + "has_single_resource", + "resource_attributes", + "resource_class", + "root", + "is_file", + "temp_dir", + "resources_by_path", + "resources_count", + "paths", + "max_in_memory", + "all_in_memory", + "all_on_disk", + "cache_dir", + "headers", + "current_header", + "codebase_attributes", + "attributes", + "counters", + "timings", + "errors", ) # the value returned if the resource is cached @@ -286,7 +284,7 @@ def __init__( ######################################################################## location = os.fsdecode(location) location = abspath(normpath(expanduser(location))) - location = location.rstrip('/\\') + location = location.rstrip("/\\") # TODO: what if is_special(location)??? assert exists(location) self.location = location @@ -294,8 +292,7 @@ def __init__( self.is_file = filetype_is_file(location) # True if this codebase root is a file or an empty directory. - self.has_single_resource = bool( - self.is_file or not os.listdir(location)) + self.has_single_resource = bool(self.is_file or not os.listdir(location)) ######################################################################## # Set up caching, summary, timing, and error info @@ -311,8 +308,11 @@ def _prepare_clean_paths(self, paths=tuple()): We convert to POSIX and ensure we have no slash at both ends. """ paths = (clean_path(p) for p in (paths or []) if p) + # we sort by path segments (e.g. essentially a topo sort) - def _sorter(p): return p.split('/') + def _sorter(p): + return p.split("/") + return sorted(paths, key=_sorter) def _setup_essentials(self, temp_dir=temp_dir, max_in_memory=10000): @@ -391,7 +391,7 @@ def _get_resource_cache_location(self, path, create_dirs=False): path = clean_path(path) # for the cached file name, we use an md5 of the path to avoid things being too long - resid = str(md5(path.encode('utf-8')).hexdigest()) + resid = str(md5(path.encode("utf-8")).hexdigest()) cache_sub_dir, cache_file_name = resid[-2:], resid parent = join(self.cache_dir, cache_sub_dir) @@ -412,7 +412,7 @@ def _build_resource_class(self, *args, **kwargs): """ # Resource sub-class to use. Configured with plugin attributes if present return attr.make_class( - name='ScannedResource', + name="ScannedResource", attrs=self.resource_attributes or {}, slots=True, bases=(Resource,), @@ -441,8 +441,7 @@ def _populate(self): # Codebase attributes to use. Configured with plugin attributes if # present. self.codebase_attributes = self._collect_codebase_attributes() - cbac = _CodebaseAttributes.from_attributes( - attributes=self.codebase_attributes) + cbac = _CodebaseAttributes.from_attributes(attributes=self.codebase_attributes) self.attributes = cbac() # Resource sub-class to use. Configured with plugin attributes if present @@ -456,7 +455,7 @@ def _populate(self): ########################################################## root = self._create_root_resource() if TRACE: - logger_debug('Codebase.populate: root:', root) + logger_debug("Codebase.populate: root:", root) if self.has_single_resource: # there is nothing else to do for a single file or a single @@ -483,7 +482,7 @@ def _create_resources_from_paths(self, root, paths): for path in paths: res_loc = join(base_location, path) if not exists(res_loc): - msg = f'ERROR: cannot populate codebase: path: {path!r} not found in {res_loc!r}' + msg = f"ERROR: cannot populate codebase: path: {path!r} not found in {res_loc!r}" self.errors.append(msg) raise Exception(path, join(base_location, path)) continue @@ -491,19 +490,17 @@ def _create_resources_from_paths(self, root, paths): # create all parents. The last parent is the one we want to use parent = root if TRACE: - logger_debug( - 'Codebase._create_resources_from_paths: parent', parent) + logger_debug("Codebase._create_resources_from_paths: parent", parent) for parent_path in get_ancestor_paths(path, include_self=False): if TRACE: logger_debug( - f' Codebase._create_resources_from_paths: parent_path: {parent_path!r}' + f" Codebase._create_resources_from_paths: parent_path: {parent_path!r}" ) if not parent_path: continue newpar = parents_by_path.get(parent_path) if TRACE: - logger_debug( - ' Codebase._create_resources_from_paths: newpar', repr(newpar)) + logger_debug(" Codebase._create_resources_from_paths: newpar", repr(newpar)) if not newpar: newpar = self._get_or_create_resource( @@ -514,7 +511,7 @@ def _create_resources_from_paths(self, root, paths): ) if not newpar: raise Exception( - f'ERROR: Codebase._create_resources_from_paths: cannot create parent for: {parent_path!r}' + f"ERROR: Codebase._create_resources_from_paths: cannot create parent for: {parent_path!r}" ) parent = newpar @@ -522,8 +519,8 @@ def _create_resources_from_paths(self, root, paths): if TRACE: logger_debug( - f' Codebase._create_resources_from_paths:', - f'created newpar: {newpar!r}', + f" Codebase._create_resources_from_paths:", + f"created newpar: {newpar!r}", ) res = self._get_or_create_resource( @@ -533,8 +530,7 @@ def _create_resources_from_paths(self, root, paths): is_file=isfile(res_loc), ) if TRACE: - logger_debug( - 'Codebase._create_resources_from_paths: resource', res) + logger_debug("Codebase._create_resources_from_paths: resource", res) def _create_resources_from_root(self, root): # without paths we walks the root location top-down @@ -547,11 +543,11 @@ def _create_resources_from_root(self, root): def err(_error): """os.walk error handler""" self.errors.append( - f'ERROR: cannot populate codebase: {_error}\n{traceback.format_exc()}' + f"ERROR: cannot populate codebase: {_error}\n{traceback.format_exc()}" ) # Walk over the directory and build the resource tree - for (top, dirs, files) in depth_walk( + for top, dirs, files in depth_walk( root_location=root.location, max_depth=self.max_depth, error_handler=err, @@ -586,7 +582,7 @@ def _create_resources(self, parent, top, dirs, files, skip_ignored=skip_ignored) is_file=is_file, ) if TRACE: - logger_debug('Codebase.create_resources:', res) + logger_debug("Codebase.create_resources:", res) yield res def _create_root_resource(self): @@ -595,8 +591,7 @@ def _create_root_resource(self): """ # we cannot recreate a root if it exists!! if self.root: - raise TypeError( - 'Root resource already exists and cannot be recreated') + raise TypeError("Root resource already exists and cannot be recreated") location = self.location name = file_name(location) @@ -605,8 +600,7 @@ def _create_root_resource(self): path = Resource.build_path(root_location=location, location=location) if TRACE: - logger_debug( - f' Codebase._create_root_resource: {path} is_file: {self.is_file}') + logger_debug(f" Codebase._create_root_resource: {path} is_file: {self.is_file}") logger_debug() root = self.resource_class( @@ -636,7 +630,7 @@ def _get_or_create_resource( """ if not parent: raise TypeError( - f'Cannot create resource without parent: name: {name!r}, path: {path!r}' + f"Cannot create resource without parent: name: {name!r}, path: {path!r}" ) # If the codebase is virtual, we provide the path @@ -647,13 +641,11 @@ def _get_or_create_resource( existing = self.get_resource(path) if existing: if TRACE: - logger_debug( - ' Codebase._get_or_create_resource: path already exists:', path) + logger_debug(" Codebase._get_or_create_resource: path already exists:", path) return existing if self._use_disk_cache_for_resource(): - cache_location = self._get_resource_cache_location( - path=path, create_dirs=True) + cache_location = self._get_resource_cache_location(path=path, create_dirs=True) else: cache_location = None @@ -666,8 +658,8 @@ def _get_or_create_resource( if TRACE: logger_debug( - f' Codebase._get_or_create_resource: with path: {path}\n' - f' name={name}, is_file={is_file}' + f" Codebase._get_or_create_resource: with path: {path}\n" + f" name={name}, is_file={is_file}" ) child = self.resource_class( @@ -698,7 +690,7 @@ def get_files_count(self): """ Return the final files counts for the codebase. """ - return self.counters.get('final:files_count', 0) + return self.counters.get("final:files_count", 0) def add_files_count_to_current_header(self): """ @@ -707,7 +699,7 @@ def add_files_count_to_current_header(self): """ files_count = self.get_files_count() current_header = self.get_or_create_current_header() - current_header.extra_data['files_count'] = files_count + current_header.extra_data["files_count"] = files_count return files_count def get_headers(self): @@ -743,10 +735,10 @@ def _use_disk_cache_for_resource(self): if TRACE: logger_debug( - f' Codebase._use_disk_cache_for_resource mode: {use_disk_cache} ' - f'on_disk: {self.all_on_disk} ' - f'in_mem: {self.all_in_memory} ' - f'max_in_mem: {self.max_in_memory}' + f" Codebase._use_disk_cache_for_resource mode: {use_disk_cache} " + f"on_disk: {self.all_on_disk} " + f"in_mem: {self.all_in_memory} " + f"max_in_mem: {self.max_in_memory}" ) return use_disk_cache @@ -763,8 +755,7 @@ def _exists_on_disk(self, path): """ path = clean_path(path) if not self._exists_in_memory(path): - cache_location = self._get_resource_cache_location( - path, create_dirs=False) + cache_location = self._get_resource_cache_location(path, create_dirs=False) if cache_location: return exists(cache_location) @@ -775,16 +766,15 @@ def get_resource(self, path): The ``path`` must be relative to the root (and including the root name as its first segment). """ - assert isinstance( - path, str), f'Invalid path: {path!r} is not a string.' + assert isinstance(path, str), f"Invalid path: {path!r} is not a string." path = clean_path(path) if TRACE: - msg = [' Codebase.get_resource:', 'path:', path] + msg = [" Codebase.get_resource:", "path:", path] if not path or path not in self.resources_by_path: - msg.append('not in resources!') + msg.append("not in resources!") else: - msg.extend(['exists on disk:', self._exists_on_disk(path)]) - msg.extend(['exists in memo:', self._exists_in_memory(path)]) + msg.extend(["exists on disk:", self._exists_on_disk(path)]) + msg.extend(["exists in memo:", self._exists_in_memory(path)]) logger_debug(*msg) # we use Codebase.CACHED_RESOURCE as a semaphore for existing but only @@ -801,11 +791,10 @@ def get_resource(self, path): pass else: # this should never happen - raise Exception( - f'get_resource: Internal error when getting {path!r}') + raise Exception(f"get_resource: Internal error when getting {path!r}") if TRACE: - logger_debug(' Resource:', res) + logger_debug(" Resource:", res) return res def save_resource(self, resource): @@ -818,7 +807,7 @@ def save_resource(self, resource): path = clean_path(resource.path) if TRACE: - logger_debug(' Codebase.save_resource:', resource) + logger_debug(" Codebase.save_resource:", resource) if resource.is_root: self.root = resource @@ -839,13 +828,12 @@ def _dump_resource(self, resource): if not cache_location: raise TypeError( - 'Resource cannot be dumped to disk and is used only' f'in memory: {resource}' + "Resource cannot be dumped to disk and is used only" f"in memory: {resource}" ) # TODO: consider messagepack or protobuf for compact/faster processing? - with open(cache_location, 'w') as cached: - cached.write(json.dumps( - resource.serialize(), check_circular=False)) + with open(cache_location, "w") as cached: + cached.write(json.dumps(resource.serialize(), check_circular=False)) # TODO: consider adding a small LRU cache in front of this for perf? def _load_resource(self, path): @@ -853,35 +841,32 @@ def _load_resource(self, path): Return a Resource with ``path`` loaded from the disk cache. """ path = clean_path(path) - cache_location = self._get_resource_cache_location( - path, create_dirs=False) + cache_location = self._get_resource_cache_location(path, create_dirs=False) if TRACE: logger_debug( - ' Codebase._load_resource: exists:', + " Codebase._load_resource: exists:", exists(cache_location), - 'cache_location:', + "cache_location:", cache_location, ) if not exists(cache_location): - raise ResourceNotInCache( - f'Failed to load Resource: {path} from {cache_location!r}') + raise ResourceNotInCache(f"Failed to load Resource: {path} from {cache_location!r}") # TODO: consider messagepack or protobuf for compact/faster processing try: - with open(cache_location, 'rb') as cached: + with open(cache_location, "rb") as cached: # TODO: Use custom json encoder to encode JSON list as a tuple # TODO: Consider using simplejson data = json.load(cached) return self.resource_class(**data) except Exception as e: - with open(cache_location, 'rb') as cached: + with open(cache_location, "rb") as cached: cached_data = cached.read() msg = ( - f'ERROR: failed to load resource from cached location: {cache_location} ' - 'with content:\n\n' + - repr(cached_data) + '\n\n' + traceback.format_exc() + f"ERROR: failed to load resource from cached location: {cache_location} " + "with content:\n\n" + repr(cached_data) + "\n\n" + traceback.format_exc() ) raise Exception(msg) from e @@ -891,13 +876,12 @@ def _remove_resource(self, resource): Does not remove children. """ if resource.is_root: - raise TypeError( - f'Cannot remove the root resource from codebase: {resource!r}') + raise TypeError(f"Cannot remove the root resource from codebase: {resource!r}") # remove from in-memory cache. The disk cache is cleared on exit. self.resources_by_path.pop(resource.path, None) if TRACE: - logger_debug('Codebase._remove_resource:', resource) + logger_debug("Codebase._remove_resource:", resource) def remove_resource(self, resource): """ @@ -905,12 +889,11 @@ def remove_resource(self, resource): codebase. Return a set of removed Resource paths. """ if TRACE: - logger_debug('Codebase.remove_resource') - logger_debug(' resource', resource) + logger_debug("Codebase.remove_resource") + logger_debug(" resource", resource) if resource.is_root: - raise TypeError( - f'Cannot remove the root resource from codebase: {resource!r}') + raise TypeError(f"Cannot remove the root resource from codebase: {resource!r}") removed_paths = set() @@ -923,7 +906,7 @@ def remove_resource(self, resource): # remove resource from parent parent = resource.parent(self) if TRACE: - logger_debug(' parent', parent) + logger_debug(" parent", parent) parent.children_names.remove(resource.name) parent.save(self) @@ -1026,7 +1009,7 @@ def update_counts(self, skip_filtered=False): try: resource._compute_children_counts(self, skip_filtered) except Exception as e: - msg = f'ERROR: cannot compute children counts for: {resource.path}' + msg = f"ERROR: cannot compute children counts for: {resource.path}" raise Exception(msg) from e def clear(self): @@ -1178,11 +1161,11 @@ class Resource(object): @property def type(self): - return 'file' if self.is_file else 'directory' + return "file" if self.is_file else "directory" @type.setter def type(self, value): - if value == 'file': + if value == "file": self.is_file = True else: self.is_file = False @@ -1207,10 +1190,10 @@ def build_path(cls, root_location, location): assert loc.startswith(root_loc) # keep the root directory name by default - root_loc = posixpath_parent(root_loc).strip('/') - path = loc.replace(root_loc, '', 1).strip('/') + root_loc = posixpath_parent(root_loc).strip("/") + path = loc.replace(root_loc, "", 1).strip("/") if TRACE: - logger_debug('build_path:', root_loc, loc, path) + logger_debug("build_path:", root_loc, loc, path) return path def get_path(self, full_root=False, strip_root=False): @@ -1282,7 +1265,7 @@ def extracted_to(self, codebase): """ Return the path this Resource archive was extracted to or None. """ - extract_path = f'{self.path}-extract' + extract_path = f"{self.path}-extract" return codebase.get_resource(extract_path) def extracted_from(self, codebase): @@ -1290,8 +1273,8 @@ def extracted_from(self, codebase): Return the path to an archive this Resource was extracted from or None. """ path = self.path - if '-extract' in path: - archive_path, _, _ = self.path.rpartition('-extract') + if "-extract" in path: + archive_path, _, _ = self.path.rpartition("-extract") return codebase.get_resource(archive_path) @classmethod @@ -1407,10 +1390,11 @@ def children(self, codebase, names=()): child_path = partial(posixpath_join, self.path) get_child = codebase.get_resource - children = [get_child(path=child_path(name)) - for name in children_names] + children = [get_child(path=child_path(name)) for name in children_names] + + def _sorter(r): + return (r.has_children(), r.name.lower(), r.name) - def _sorter(r): return (r.has_children(), r.name.lower(), r.name) return sorted((c for c in children if c), key=_sorter) def has_parent(self): @@ -1515,35 +1499,34 @@ def to_dict( return res if with_info: - res['name'] = self.name - res['base_name'] = self.base_name - res['extension'] = self.extension - res['size'] = self.size + res["name"] = self.name + res["base_name"] = self.base_name + res["extension"] = self.extension + res["size"] = self.size # exclude by default all of the "standard", default Resource fields self_fields_filter = attr.filters.exclude(*attr.fields(Resource)) # this will catch every attribute that has been added dynamically, such # as scan-provided resource_attributes - other_data = attr.asdict( - self, filter=self_fields_filter, dict_factory=dict) + other_data = attr.asdict(self, filter=self_fields_filter, dict_factory=dict) # FIXME: make a deep copy of the data first!!!! # see https://github.com/aboutcode-org/scancode-toolkit/issues/1199 res.update(other_data) if with_timing: - res['scan_time'] = self.scan_time or 0 - res['scan_timings'] = self.scan_timings or dict() + res["scan_time"] = self.scan_time or 0 + res["scan_timings"] = self.scan_timings or dict() if with_info: - res['files_count'] = self.files_count - res['dirs_count'] = self.dirs_count - res['size_count'] = self.size_count + res["files_count"] = self.files_count + res["dirs_count"] = self.dirs_count + res["size_count"] = self.size_count - res['scan_errors'] = self.scan_errors + res["scan_errors"] = self.scan_errors if TRACE: - logger_debug('Resource.to_dict:', res) + logger_debug("Resource.to_dict:", res) return res def serialize(self): @@ -1554,11 +1537,11 @@ def serialize(self): """ # we save all fields, not just the one in .to_dict() serializable = attr.asdict(self) - serializable['name'] = self.name + serializable["name"] = self.name if self.location: - serializable['location'] = self.location + serializable["location"] = self.location if self.cache_location: - serializable['cache_location'] = self.cache_location + serializable["cache_location"] = self.cache_location return serializable @@ -1566,11 +1549,11 @@ def clean_path(path): """ Return a cleaned and normalized POSIX ``path``. """ - path = path or '' + path = path or "" # convert to posix and ensure we have no slash at both ends - path = posixpath_normpath(path.replace('\\', '/').strip('/')) - if path == '.': - path = '' + path = posixpath_normpath(path.replace("\\", "/").strip("/")) + if path == ".": + path = "" return path @@ -1593,11 +1576,11 @@ def strip_first_path_segment(path): '' """ path = clean_path(path) - if '/' in path: - _root, _, path = path.partition('/') + if "/" in path: + _root, _, path = path.partition("/") return path else: - return '' + return "" def get_codebase_cache_dir(temp_dir): @@ -1609,13 +1592,12 @@ def get_codebase_cache_dir(temp_dir): from commoncode.fileutils import get_temp_dir from commoncode.timeutils import time2tstamp - prefix = 'scancode-codebase-' + time2tstamp() + '-' + prefix = "scancode-codebase-" + time2tstamp() + "-" return get_temp_dir(base_dir=temp_dir, prefix=prefix) @attr.s(slots=True) class _CodebaseAttributes(object): - def to_dict(self): return attr.asdict(self, dict_factory=dict) @@ -1626,7 +1608,7 @@ def from_attributes(cls, attributes): ``attributes`` mapping of "attr" attributes. """ return attr.make_class( - name='CodebaseAttributes', + name="CodebaseAttributes", attrs=attributes or {}, slots=True, bases=(_CodebaseAttributes,), @@ -1660,11 +1642,10 @@ def build_attributes_defs(mapping, ignored_keys=()): class VirtualCodebase(Codebase): - __slots__ = ( # TRUE iff the loaded virtual codebase has file information - 'with_info', - 'has_single_resource', + "with_info", + "has_single_resource", ) def __init__( @@ -1685,7 +1666,7 @@ def __init__( `max_depth`, if passed, will be ignored as VirtualCodebase will be using the depth of the original scan. """ - logger_debug(f'VirtualCodebase: new from: {location!r}') + logger_debug(f"VirtualCodebase: new from: {location!r}") self._setup_essentials(temp_dir, max_in_memory) @@ -1706,7 +1687,6 @@ def _get_scan_data_helper(self, location): try: return json.loads(location) except: - location = abspath(normpath(expanduser(location))) with open(location) as f: scan_data = json.load(f) @@ -1736,23 +1716,22 @@ def _get_scan_data(self, location): combined_scan_data = dict(headers=[], files=[]) for idx, loc in enumerate(location, 1): scan_data = self._get_scan_data_helper(loc) - headers = scan_data.get('headers') + headers = scan_data.get("headers") if headers: - combined_scan_data['headers'].extend(headers) - files = scan_data.get('files') + combined_scan_data["headers"].extend(headers) + files = scan_data.get("files") if files: for f in files: - f['path'] = posixpath_join( - f'codebase-{idx}', clean_path(f['path'])) - combined_scan_data['files'].extend(files) + f["path"] = posixpath_join(f"codebase-{idx}", clean_path(f["path"])) + combined_scan_data["files"].extend(files) else: raise Exception( f'Input file is missing a "files" (aka. resources) section to load: {loc}' ) - combined_scan_data['headers'] = sorted( - combined_scan_data['headers'], - key=lambda x: x['start_timestamp'], + combined_scan_data["headers"] = sorted( + combined_scan_data["headers"], + key=lambda x: x["start_timestamp"], ) return combined_scan_data @@ -1790,7 +1769,7 @@ def _collect_codebase_attributes(self, scan_data, *args, **kwargs): all_attributes = ( build_attributes_defs( mapping=scan_data, - ignored_keys=('headers', 'files'), + ignored_keys=("headers", "files"), ) or {} ) @@ -1811,7 +1790,7 @@ def _build_resource_class(self, sample_resource_data, *args, **kwargs): standard_res_attributes = set(f.name for f in attr.fields(Resource)) # add these properties since they are fields but are serialized - properties = set(['type', 'base_name', 'extension']) + properties = set(["type", "base_name", "extension"]) standard_res_attributes.update(properties) # We collect attributes that are not in standard_res_attributes already @@ -1829,7 +1808,7 @@ def _build_resource_class(self, sample_resource_data, *args, **kwargs): # Create the Resource class with the desired attributes return attr.make_class( - name='ScannedResource', + name="ScannedResource", attrs=all_res_attributes or dict(), slots=True, bases=(Resource,), @@ -1847,7 +1826,7 @@ def _populate(self, scan_data): """ # Collect headers ########################################################## - headers = scan_data.get('headers') or [] + headers = scan_data.get("headers") or [] headers = [Header.from_dict(**hle) for hle in headers] self.headers = headers @@ -1856,8 +1835,7 @@ def _populate(self, scan_data): # Codebase attributes to use. Configured with scan_data and plugin # attributes if present. self.codebase_attributes = self._collect_codebase_attributes(scan_data) - cbac = _CodebaseAttributes.from_attributes( - attributes=self.codebase_attributes) + cbac = _CodebaseAttributes.from_attributes(attributes=self.codebase_attributes) self.attributes = cbac() # now populate top level codebase attributes @@ -1869,14 +1847,14 @@ def _populate(self, scan_data): setattr(self.attributes, attr_name, value) ########################################################## - files_data = scan_data.get('files') + files_data = scan_data.get("files") if not files_data: raise Exception('Input has no "files" top-level scan results.') if len(files_data) == 1: # we will shortcut to populate the codebase with a single root resource self.has_single_resource = True - root_is_file = files_data[0].get('type') == 'file' + root_is_file = files_data[0].get("type") == "file" else: root_is_file = False @@ -1908,9 +1886,9 @@ def _populate(self, scan_data): for fdata in files_data: sample_resource_data_update(fdata) - segments = fdata['path'].split('/') + segments = fdata["path"].split("/") root_names_add(segments[0]) - fdata['path_segments'] = segments + fdata["path_segments"] = segments # Resource sub-class to use. Configured with all known scanned file # attributes and plugin attributes if present @@ -1921,13 +1899,13 @@ def _populate(self, scan_data): self.with_info = any( a in sample_resource_data for a in ( - 'name', - 'base_name', - 'extension', - 'size', - 'files_count', - 'dirs_count', - 'size_count', + "name", + "base_name", + "extension", + "size", + "files_count", + "dirs_count", + "size_count", ) ) @@ -1935,28 +1913,28 @@ def _populate(self, scan_data): # Create root resource first ########################################################## if not root_names: - raise Exception('Unable to find root for codebase.') + raise Exception("Unable to find root for codebase.") len_root_names = len(root_names) if len_root_names == 1: root_path = root_names.pop() needs_new_virtual_root = False elif len_root_names > 1 or multiple_inputs: - root_path = 'virtual_root' + root_path = "virtual_root" needs_new_virtual_root = True if needs_new_virtual_root: for fdata in files_data: - rpath = fdata['path'] - fdata['path'] = posixpath_join(root_path, rpath) - fdata['path_segments'].insert(0, root_path) + rpath = fdata["path"] + fdata["path"] = posixpath_join(root_path, rpath) + fdata["path_segments"].insert(0, root_path) root_data = self._create_empty_resource_data() if self.has_single_resource: # single resource with one or more segments rdata = files_data[0] - root_path = rdata['path'] + root_path = rdata["path"] rdata = remove_properties_and_basics(rdata) root_data.update(rdata) @@ -1973,14 +1951,14 @@ def _populate(self, scan_data): setattr(root, name, value) if TRACE: - logger_debug('VirtualCodebase.populate: root:', root) + logger_debug("VirtualCodebase.populate: root:", root) # TODO: report error if filtering the root with a paths? self.save_resource(root) if self.has_single_resource: if TRACE: - logger_debug('VirtualCodebase.populate: with single resource.') + logger_debug("VirtualCodebase.populate: with single resource.") return all_paths = None @@ -1994,21 +1972,21 @@ def _populate(self, scan_data): # Note that we do not know the ordering there. # Therefore we sort in place by path segments - files_data.sort(key=itemgetter('path_segments')) + files_data.sort(key=itemgetter("path_segments")) # We create directories that exist in the scan or create these that # exist only in paths duplicated_paths = set() last_path = None for fdata in files_data: - path = fdata.get('path') + path = fdata.get("path") # skip the ones we did not request if all_paths and path not in all_paths: continue # these are no longer needed - path_segments = fdata.pop('path_segments') + path_segments = fdata.pop("path_segments") if not last_path: last_path = path @@ -2017,11 +1995,11 @@ def _populate(self, scan_data): else: last_path = path - name = fdata.get('name', None) or None + name = fdata.get("name", None) or None if not name: name = file_name(path) - is_file = fdata.get('type', 'file') == 'file' + is_file = fdata.get("type", "file") == "file" parent = self._get_parent_directory(path_segments=path_segments) resource = self._get_or_create_resource( @@ -2040,8 +2018,8 @@ def _populate(self, scan_data): if duplicated_paths: raise Exception( - 'Illegal combination of VirtualCode multiple inputs: ' - f'duplicated paths: {list(duplicated_paths)}', + "Illegal combination of VirtualCode multiple inputs: " + f"duplicated paths: {list(duplicated_paths)}", ) def _get_parent_directory(self, path_segments): @@ -2077,14 +2055,12 @@ def _create_root_resource(self, name, path, is_file): """ # we cannot recreate a root if it exists!! if self.root: - raise TypeError( - 'Root resource already exists and cannot be recreated') + raise TypeError("Root resource already exists and cannot be recreated") path = clean_path(path) if TRACE: - logger_debug( - f' VirtualCodebase._create_root_resource: {path!r} is_file: {is_file}') + logger_debug(f" VirtualCodebase._create_root_resource: {path!r} is_file: {is_file}") root = self.resource_class( name=name, @@ -2102,8 +2078,7 @@ def _create_root_resource(self, name, path, is_file): return root -KNOW_PROPS = set(['type', 'base_name', 'extension', - 'path', 'name', 'path_segments']) +KNOW_PROPS = set(["type", "base_name", "extension", "path", "name", "path_segments"]) def remove_properties_and_basics(resource_data): @@ -2128,10 +2103,10 @@ def get_ancestor_paths(path, include_self=False): >>> assert results == [], results """ assert path - segments = path.split('/') + segments = path.split("/") if not include_self: segments = segments[:-1] subpath = [] for segment in segments: subpath.append(segment) - yield '/'.join(subpath) + yield "/".join(subpath) diff --git a/src/commoncode/system.py b/src/commoncode/system.py index a2a43d2b..0566a6c0 100644 --- a/src/commoncode/system.py +++ b/src/commoncode/system.py @@ -16,22 +16,22 @@ def os_arch(): """ Return a tuple for the current the OS and architecture. """ - if sys.maxsize > 2 ** 32: - arch = '64' + if sys.maxsize > 2**32: + arch = "64" else: - arch = '32' + arch = "32" sys_platform = str(sys.platform).lower() - if sys_platform.startswith('linux'): - os = 'linux' - elif 'win32' in sys_platform: - os = 'win' - elif 'darwin' in sys_platform: - os = 'mac' - elif 'freebsd' in sys_platform: - os = 'freebsd' + if sys_platform.startswith("linux"): + os = "linux" + elif "win32" in sys_platform: + os = "win" + elif "darwin" in sys_platform: + os = "mac" + elif "freebsd" in sys_platform: + os = "freebsd" else: - raise Exception('Unsupported OS/platform %r' % sys_platform) + raise Exception("Unsupported OS/platform %r" % sys_platform) return os, arch @@ -39,17 +39,17 @@ def os_arch(): # OS/Arch # current_os, current_arch = os_arch() -on_windows = current_os == 'win' -on_windows_32 = on_windows and current_arch == '32' -on_windows_64 = on_windows and current_arch == '64' -on_mac = current_os == 'mac' -on_linux = current_os == 'linux' -on_freebsd = current_os == 'freebsd' +on_windows = current_os == "win" +on_windows_32 = on_windows and current_arch == "32" +on_windows_64 = on_windows and current_arch == "64" +on_mac = current_os == "mac" +on_linux = current_os == "linux" +on_freebsd = current_os == "freebsd" on_posix = not on_windows and (on_mac or on_linux or on_freebsd) -current_os_arch = '%(current_os)s-%(current_arch)s' % locals() -noarch = 'noarch' -current_os_noarch = '%(current_os)s-%(noarch)s' % locals() +current_os_arch = "%(current_os)s-%(current_arch)s" % locals() +noarch = "noarch" +current_os_noarch = "%(current_os)s-%(noarch)s" % locals() del os_arch @@ -61,10 +61,11 @@ def is_on_macos_14_or_higher(): filesystem encodings. """ import platform + macos_ver = platform.mac_ver() macos_ver = macos_ver[0] - macos_ver = macos_ver.split('.') - return macos_ver > ['10', '14'] + macos_ver = macos_ver.split(".") + return macos_ver > ["10", "14"] on_macos_14_or_higher = is_on_macos_14_or_higher() @@ -77,7 +78,8 @@ def is_on_macos_arm64(): Return True if the current OS is macOS running on Apple Silicon. """ import platform - return on_mac and platform.machine() == 'arm64' + + return on_mac and platform.machine() == "arm64" on_macos_arm64 = is_on_macos_arm64() @@ -85,14 +87,15 @@ def is_on_macos_arm64(): del is_on_macos_arm64 -def is_on_ubuntu_22(os_release_location='/etc/os-release'): +def is_on_ubuntu_22(os_release_location="/etc/os-release"): """ Return True if the current OS is Ubuntu 22.XX. """ if not on_linux: return False os_release_info = parse_os_release(os_release_location) - return os_release_info['ID'] == 'ubuntu' and '22' in os_release_info['VERSION_ID'] + return os_release_info["ID"] == "ubuntu" and "22" in os_release_info["VERSION_ID"] + try: on_ubuntu_22 = is_on_ubuntu_22() @@ -126,11 +129,11 @@ def has_case_sensitive_fs(): # Shared library file extensions # if on_windows: - lib_ext = '.dll' + lib_ext = ".dll" if on_mac: - lib_ext = '.dylib' + lib_ext = ".dylib" if on_linux or on_freebsd: - lib_ext = '.so' + lib_ext = ".so" # # Python versions @@ -157,5 +160,6 @@ def has_case_sensitive_fs(): # The system does not display the Windows Error Reporting dialog. if on_windows: import ctypes + # 3 is SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX ctypes.windll.kernel32.SetErrorMode(3) # @UndefinedVariable diff --git a/src/commoncode/testcase.py b/src/commoncode/testcase.py index 6ce7b582..b24d4d1c 100644 --- a/src/commoncode/testcase.py +++ b/src/commoncode/testcase.py @@ -12,15 +12,15 @@ import shutil import stat import sys -from os import path from collections import defaultdict from itertools import chain +from os import path from unittest import TestCase as TestCaseClass import saneyaml -from commoncode import fileutils from commoncode import filetype +from commoncode import fileutils from commoncode.archive import extract_tar from commoncode.archive import extract_tar_raw from commoncode.archive import extract_tar_uni @@ -42,13 +42,9 @@ def to_os_native_path(path): """ Normalize a path to use the native OS path separator. """ - OS_PATH_SEP = '\\' if on_windows else '/' + OS_PATH_SEP = "\\" if on_windows else "/" - return ( - path.replace('/', OS_PATH_SEP) - .replace(u'\\', OS_PATH_SEP) - .rstrip(OS_PATH_SEP) - ) + return path.replace("/", OS_PATH_SEP).replace("\\", OS_PATH_SEP).rstrip(OS_PATH_SEP) def get_test_loc( @@ -64,6 +60,7 @@ def get_test_loc( """ if debug: import inspect + caller = inspect.stack()[1][3] print('\nget_test_loc,%(caller)s,"%(test_path)s","%(test_data_dir)s"' % locals()) @@ -71,15 +68,18 @@ def get_test_loc( assert test_data_dir if not path.exists(test_data_dir): - raise IOError("[Errno 2] No such directory: test_data_dir not found:" - " '%(test_data_dir)s'" % locals()) + raise IOError( + "[Errno 2] No such directory: test_data_dir not found:" + " '%(test_data_dir)s'" % locals() + ) tpath = to_os_native_path(test_path) test_loc = path.abspath(path.join(test_data_dir, tpath)) if must_exist and not path.exists(test_loc): - raise IOError("[Errno 2] No such file or directory: " - "test_path not found: '%(test_loc)s'" % locals()) + raise IOError( + "[Errno 2] No such file or directory: " "test_path not found: '%(test_loc)s'" % locals() + ) return test_loc @@ -90,6 +90,7 @@ class FileDrivenTesting(object): temporary test resources and doing file-based assertions. This can be used as a standalone object if needed. """ + test_data_dir = None def get_test_loc(self, test_path, copy=False, debug=False, must_exist=True): @@ -104,6 +105,7 @@ def get_test_loc(self, test_path, copy=False, debug=False, must_exist=True): test_data_dir = self.test_data_dir if debug: import inspect + caller = inspect.stack()[1][3] print('\nself.get_test_loc,%(caller)s,"%(test_path)s"' % locals()) @@ -129,16 +131,16 @@ def get_test_loc(self, test_path, copy=False, debug=False, must_exist=True): test_loc = target_dir return test_loc - def get_temp_file(self, extension=None, dir_name='td', file_name='tf'): + def get_temp_file(self, extension=None, dir_name="td", file_name="tf"): """ Return a unique new temporary file location to a non-existing temporary file that can safely be created without a risk of name collision. """ if extension is None: - extension = '.txt' + extension = ".txt" - if extension and not extension.startswith('.'): - extension = '.' + extension + if extension and not extension.startswith("."): + extension = "." + extension file_name = file_name + extension temp_dir = self.get_temp_dir(dir_name) @@ -156,19 +158,19 @@ def get_temp_dir(self, sub_dir_path=None): global test_run_temp_dir if not test_run_temp_dir: import tempfile + test_tmp_root_dir = tempfile.gettempdir() # now we add a space in the path for testing path with spaces test_run_temp_dir = fileutils.get_temp_dir( - base_dir=test_tmp_root_dir, prefix='scancode-tk-tests -') + base_dir=test_tmp_root_dir, prefix="scancode-tk-tests -" + ) - test_run_temp_subdir = fileutils.get_temp_dir( - base_dir=test_run_temp_dir, prefix='') + test_run_temp_subdir = fileutils.get_temp_dir(base_dir=test_run_temp_dir, prefix="") if sub_dir_path: # create a sub directory hierarchy if requested sub_dir_path = to_os_native_path(sub_dir_path) - test_run_temp_subdir = path.join( - test_run_temp_subdir, sub_dir_path) + test_run_temp_subdir = path.join(test_run_temp_subdir, sub_dir_path) fileutils.create_dir(test_run_temp_subdir) return test_run_temp_subdir @@ -176,7 +178,7 @@ def remove_vcs(self, test_dir): """ Remove some version control directories and some temp editor files. """ - vcses = ('CVS', '.svn', '.git', '.hg') + vcses = ("CVS", ".svn", ".git", ".hg") for root, dirs, files in os.walk(test_dir): for vcs_dir in vcses: if vcs_dir in dirs: @@ -187,8 +189,9 @@ def remove_vcs(self, test_dir): shutil.rmtree(path.join(root, vcs_dir), False) # editors temp file leftovers - tilde_files = [path.join(root, file_loc) - for file_loc in files if file_loc.endswith('~')] + tilde_files = [ + path.join(root, file_loc) for file_loc in files if file_loc.endswith("~") + ] for tf in tilde_files: os.remove(tf) @@ -199,7 +202,7 @@ def __extract(self, test_path, extract_func=None, verbatim=False): archive file has been extracted using extract_func. If `verbatim` is True preserve the permissions. """ - assert test_path and test_path != '' + assert test_path and test_path != "" test_path = to_os_native_path(test_path) target_path = path.basename(test_path) target_dir = self.get_temp_dir(target_path) @@ -238,8 +241,7 @@ def phase3(self): Find out differences between common files. Ensure we are using content comparison, not os.stat-only. """ - comp = filecmp.cmpfiles(self.left, self.right, - self.common_files, shallow=False) + comp = filecmp.cmpfiles(self.left, self.right, self.common_files, shallow=False) self.same_files, self.diff_files, self.funny_files = comp @@ -249,13 +251,11 @@ def is_same(dir1, dir2): Return False if they differ, True is they are the same. """ compared = dircmp(dir1, dir2) - if (compared.left_only or compared.right_only or compared.diff_files - or compared.funny_files): + if compared.left_only or compared.right_only or compared.diff_files or compared.funny_files: return False for subdir in compared.common_dirs: - if not is_same(path.join(dir1, subdir), - path.join(dir2, subdir)): + if not is_same(path.join(dir1, subdir), path.join(dir2, subdir)): return False return True @@ -265,14 +265,14 @@ def file_cmp(file1, file2, ignore_line_endings=False): Compare two files content. Return False if they differ, True is they are the same. """ - with open(file1, 'rb') as f1: + with open(file1, "rb") as f1: f1c = f1.read() if ignore_line_endings: - f1c = b'\n'.join(f1c.splitlines(False)) - with open(file2, 'rb') as f2: + f1c = b"\n".join(f1c.splitlines(False)) + with open(file2, "rb") as f2: f2c = f2.read() if ignore_line_endings: - f2c = b'\n'.join(f2c.splitlines(False)) + f2c = b"\n".join(f2c.splitlines(False)) assert f2c == f1c @@ -306,6 +306,7 @@ def make_non_executable(location): current_stat = stat.S_IMODE(os.lstat(location).st_mode) os.chmod(location, current_stat & ~stat.S_IEXEC) + def get_test_file_pairs(test_dir, template_to_generate_missing_yaml=None): """ Yield tuples of (data_file, test_file) from a test data `test_dir` directory. @@ -324,16 +325,16 @@ def get_test_file_pairs(test_dir, template_to_generate_missing_yaml=None): for top, _, files in os.walk(test_dir): for tfile in files: - if tfile.endswith('~'): + if tfile.endswith("~"): continue file_path = path.abspath(path.join(top, tfile)) - if tfile.endswith('.yml'): + if tfile.endswith(".yml"): data_file_path = file_path - test_file_path = file_path.replace('.yml', '') + test_file_path = file_path.replace(".yml", "") else: test_file_path = file_path - data_file_path = test_file_path + '.yml' + data_file_path = test_file_path + ".yml" if not path.exists(test_file_path): dangling_test_files.add(test_file_path) @@ -348,11 +349,13 @@ def get_test_file_pairs(test_dir, template_to_generate_missing_yaml=None): # ensure that we haev no dangling files if dangling_test_files or dangling_data_files: - msg = ['Dangling missing test files without a YAML data file:'] + \ - sorted(dangling_test_files) - msg += ['Dangling missing YAML data files without a test file'] + \ - sorted(dangling_data_files) - msg = '\n'.join(msg) + msg = ["Dangling missing test files without a YAML data file:"] + sorted( + dangling_test_files + ) + msg += ["Dangling missing YAML data files without a test file"] + sorted( + dangling_data_files + ) + msg = "\n".join(msg) print(msg) raise Exception(msg) @@ -360,28 +363,29 @@ def get_test_file_pairs(test_dir, template_to_generate_missing_yaml=None): diff = set(data_files.keys()).symmetric_difference(set(test_files.keys())) if diff: msg = [ - 'Orphaned copyright test file(s) found: ' - 'test file without its YAML test data file ' - 'or YAML test data file without its test file.'] + sorted(diff) - msg = '\n'.join(msg) + "Orphaned copyright test file(s) found: " + "test file without its YAML test data file " + "or YAML test data file without its test file." + ] + sorted(diff) + msg = "\n".join(msg) print(msg) raise Exception(msg) # ensure that test file paths are unique when you ignore case # we use the file names as test method names (and we have Windows that's # case insensitive - dupes = list(chain.from_iterable( - paths for paths in paths_ignoring_case.values() if len(paths) != 1)) + dupes = list( + chain.from_iterable(paths for paths in paths_ignoring_case.values() if len(paths) != 1) + ) if dupes: - msg = [ - 'Non unique test/data file(s) found when ignoring case!'] + sorted(dupes) + msg = ["Non unique test/data file(s) found when ignoring case!"] + sorted(dupes) - msg = '\n'.join(msg) + msg = "\n".join(msg) print(msg) raise Exception(msg) for test_file in test_files: - yield test_file + '.yml', test_file + yield test_file + ".yml", test_file def check_against_expected_json_file(results, expected_file, regen=False): @@ -393,8 +397,8 @@ def check_against_expected_json_file(results, expected_file, regen=False): This is convenient for updating tests expectations. But use with caution. """ if regen: - with open(expected_file, 'w') as reg: - json.dump(results, reg, indent=2, separators=(',', ': ')) + with open(expected_file, "w") as reg: + json.dump(results, reg, indent=2, separators=(",", ": ")) expected = results else: with open(expected_file) as exp: diff --git a/src/commoncode/text.py b/src/commoncode/text.py index b2f4ea1c..ab56b0ae 100644 --- a/src/commoncode/text.py +++ b/src/commoncode/text.py @@ -11,8 +11,8 @@ import re import unicodedata -from text_unidecode import unidecode from bs4.dammit import UnicodeDammit +from text_unidecode import unidecode """ A text processing module providing functions to process and prepare text @@ -49,24 +49,24 @@ def foldcase(text): def nopunc(): - return re.compile(r'[\W_]', re.MULTILINE | re.UNICODE) + return re.compile(r"[\W_]", re.MULTILINE | re.UNICODE) def nopunctuation(text): - u""" + """ Replaces any non alphanum symbol (i.e. punctuation) in text with space. Preserve the characters offsets by replacing punctuation with spaces. Warning: this also drops line endings. """ if not isinstance(text, str): text = as_unicode(text) - return re.sub(nopunc(), ' ', text) + return re.sub(nopunc(), " ", text) -CR = '\r' -LF = '\n' +CR = "\r" +LF = "\n" CRLF = CR + LF -CRLF_NO_CR = ' ' + LF +CRLF_NO_CR = " " + LF def unixlinesep(text, preserve=False): @@ -85,7 +85,7 @@ def nolinesep(text): """ if not isinstance(text, str): text = as_unicode(text) - return text.replace(CR, ' ').replace(LF, ' ') + return text.replace(CR, " ").replace(LF, " ") def toascii(s, translit=False): @@ -110,11 +110,11 @@ def toascii(s, translit=False): if translit: converted = unidecode(s) else: - converted = unicodedata.normalize('NFKD', s) + converted = unicodedata.normalize("NFKD", s) - converted = converted.replace('[?]', '_') - converted = converted.encode('ascii', 'ignore') - return converted.decode('ascii') + converted = converted.replace("[?]", "_") + converted = converted.encode("ascii", "ignore") + return converted.decode("ascii") def python_safe_name(s): @@ -126,9 +126,9 @@ def python_safe_name(s): s = toascii(s) s = foldcase(s) s = nopunctuation(s) - s = s.replace(' ', '_') - s = '_'.join(s.split()) - s = s.strip('_') + s = s.replace(" ", "_") + s = "_".join(s.split()) + s = s.strip("_") return s @@ -138,9 +138,9 @@ def as_unicode(s): """ if isinstance(s, str): return s - if s == b'': - return u'' + if s == b"": + return "" if not s: return s - assert isinstance(s, bytes), 's must be bytes but is: {}'.format(s) + assert isinstance(s, bytes), "s must be bytes but is: {}".format(s) return UnicodeDammit(s).markup diff --git a/src/commoncode/timeutils.py b/src/commoncode/timeutils.py index 0bcb0116..bab8dad7 100644 --- a/src/commoncode/timeutils.py +++ b/src/commoncode/timeutils.py @@ -25,7 +25,7 @@ def utcoffset(self, dt): # NOQA return None def tzname(self, dt): # NOQA - return 'UTC' + return "UTC" def dst(self, dt): # NOQA return None @@ -61,7 +61,7 @@ def time2tstamp(dt=None, path_safe=True): datim = dt or datetime.utcnow() iso = datim.isoformat() if path_safe: - iso = iso.replace(':', '').replace('/', '_') + iso = iso.replace(":", "").replace("/", "_") return iso @@ -70,11 +70,11 @@ def tstamp2time(stamp): Convert a UTC timestamp to a datetime object. """ # handle both basic and extended formats - tformat = '%Y-%m-%dT%H%M%S' if stamp[4] == '-' else '%Y%m%dT%H%M%S' + tformat = "%Y-%m-%dT%H%M%S" if stamp[4] == "-" else "%Y%m%dT%H%M%S" # normalize - dt_ms = stamp.strip().replace('Z', '').replace(':', '') + dt_ms = stamp.strip().replace("Z", "").replace(":", "") - dt_ms = dt_ms.split('.') + dt_ms = dt_ms.split(".") isodatim = dt_ms[0] datim = datetime.strptime(isodatim, tformat) # all stamps must be UTC diff --git a/src/commoncode/urn.py b/src/commoncode/urn.py index 37d9fa9c..32dce797 100644 --- a/src/commoncode/urn.py +++ b/src/commoncode/urn.py @@ -75,21 +75,21 @@ class URNValidationError(Exception): # Describes the URN schema for each object type URN_SCHEMAS = { - 'license': { - 'object': 'license', - 'fields': ['key'], + "license": { + "object": "license", + "fields": ["key"], }, - 'owner': { - 'object': 'owner', - 'fields': ['name'], + "owner": { + "object": "owner", + "fields": ["name"], }, - 'component': { - 'object': 'component', - 'fields': ['name', 'version'], + "component": { + "object": "component", + "fields": ["name", "version"], }, - 'product': { - 'object': 'product', - 'fields': ['name', 'version'], + "product": { + "object": "product", + "fields": ["name", "version"], }, } @@ -107,13 +107,13 @@ def encode(object_type, **fields): # case is not significant for the object type object_type = object_type.strip().lower() - urn_prefix = 'urn:dje:{0}:'.format(quote_plus(object_type)) + urn_prefix = "urn:dje:{0}:".format(quote_plus(object_type)) - object_fields = URN_SCHEMAS[object_type]['fields'] + object_fields = URN_SCHEMAS[object_type]["fields"] # leading and trailing white spaces are not significant # each URN part is encoded individually BEFORE assembling the URN encoded_fields = [quote_plus(fields[f].strip()) for f in object_fields] - encoded_fields = ':'.join(encoded_fields) + encoded_fields = ":".join(encoded_fields) return urn_prefix + encoded_fields @@ -122,23 +122,23 @@ def decode(urn): Decode a URN and return the object_type and a mapping of field/values. Raise URNValidationError on errors. """ - segments = [unquote_plus(p) for p in urn.split(':')] + segments = [unquote_plus(p) for p in urn.split(":")] - if not segments[0] == ('urn'): + if not segments[0] == ("urn"): raise URNValidationError("Invalid URN prefix. Expected 'urn'.") - if not segments[1] == ('dje'): + if not segments[1] == ("dje"): raise URNValidationError("Invalid URN namespace. Expected 'dje'.") # object type is always lowercase object_type = segments[2].lower() if object_type not in URN_SCHEMAS: - raise URNValidationError('Unsupported URN object type.') + raise URNValidationError("Unsupported URN object type.") fields = segments[3:] - schema_fields = URN_SCHEMAS[object_type]['fields'] + schema_fields = URN_SCHEMAS[object_type]["fields"] if len(schema_fields) != len(fields): - raise URNValidationError('Invalid number of fields in URN.') + raise URNValidationError("Invalid number of fields in URN.") decoded_fields = dict(zip(schema_fields, fields)) return object_type, decoded_fields diff --git a/src/commoncode/version.py b/src/commoncode/version.py index 83cb2460..a99ac8d6 100644 --- a/src/commoncode/version.py +++ b/src/commoncode/version.py @@ -12,29 +12,31 @@ def VERSION_PATTERNS_REGEX(): - return [re.compile(x) for x in [ - # Eclipse features - r'v\d+\.feature\_(\d+\.){1,3}\d+', - - # Common version patterns - r'(M?(v\d+(\-|\_))?\d+\.){1,3}\d+[A-Za-z0-9]*((\.|\-|_|~)' - r'(b|B|rc|r|v|RC|alpha|beta|BETA|M|m|pre|vm|G)?\d+((\-|\.)\d+)?)?' - r'((\.|\-)(((alpha|dev|beta|rc|FINAL|final|pre)(\-|\_)\d+[A-Za-z]?' - r'(\-RELEASE)?)|alpha|dev(\.\d+\.\d+)?' - r'|beta|BETA|final|FINAL|release|fixed|(cr\d(\_\d*)?)))?', + return [ + re.compile(x) + for x in [ + # Eclipse features + r"v\d+\.feature\_(\d+\.){1,3}\d+", + # Common version patterns + r"(M?(v\d+(\-|\_))?\d+\.){1,3}\d+[A-Za-z0-9]*((\.|\-|_|~)" + r"(b|B|rc|r|v|RC|alpha|beta|BETA|M|m|pre|vm|G)?\d+((\-|\.)\d+)?)?" + r"((\.|\-)(((alpha|dev|beta|rc|FINAL|final|pre)(\-|\_)\d+[A-Za-z]?" + r"(\-RELEASE)?)|alpha|dev(\.\d+\.\d+)?" + r"|beta|BETA|final|FINAL|release|fixed|(cr\d(\_\d*)?)))?", # - r'[A-Za-z]?(\d+\_){1,3}\d+\_?[A-Za-z]{0,2}\d+', + r"[A-Za-z]?(\d+\_){1,3}\d+\_?[A-Za-z]{0,2}\d+", # - r'(b|rc|r|v|RC|alpha|beta|BETA|M|m|pre|revision-)\d+(\-\d+)?', + r"(b|rc|r|v|RC|alpha|beta|BETA|M|m|pre|revision-)\d+(\-\d+)?", # - r'current|previous|latest|alpha|beta', + r"current|previous|latest|alpha|beta", # - r'\d{4}-\d{2}-\d{2}', + r"\d{4}-\d{2}-\d{2}", # - r'(\d(\-|\_)){1,2}\d', + r"(\d(\-|\_)){1,2}\d", # - r'\d{5,14}', - ]] + r"\d{5,14}", + ] + ] def hint(path): @@ -43,7 +45,7 @@ def hint(path): the version does not start with v. """ for pattern in VERSION_PATTERNS_REGEX(): - segments = path.split('/') + segments = path.split("/") # skip the first path segment unless there's only one segment first_segment = 1 if len(segments) > 1 else 0 interesting_segments = segments[first_segment:] @@ -53,8 +55,8 @@ def hint(path): if version: v = version.group(0) # prefix with v space - if not v.lower().startswith('v'): - v = f'v {v}' + if not v.lower().startswith("v"): + v = f"v {v}" return v @@ -117,7 +119,7 @@ def is_moslty_num(s): return False -NameVersion = namedtuple('NameVersion', 'name, version') +NameVersion = namedtuple("NameVersion", "name, version") def get_jar_nv(filename): diff --git a/tests/test_cliutils.py b/tests/test_cliutils.py index a1d88ce7..725eac91 100644 --- a/tests/test_cliutils.py +++ b/tests/test_cliutils.py @@ -6,118 +6,129 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -from commoncode.cliutils import PluggableCommandLineOption +import os + +import click +from click.testing import CliRunner + from commoncode.cliutils import GroupedHelpCommand +from commoncode.cliutils import PluggableCommandLineOption from commoncode.cliutils import fixed_width_file_name from commoncode.testcase import FileDrivenTesting -from click.testing import CliRunner -import os -import click click.disable_unicode_literals_warning = True class TestFixedWidthFilename(FileDrivenTesting): - def test_fixed_width_file_name_with_file_name_larger_than_max_length_is_shortened(self): - test = fixed_width_file_name('0123456789012345678901234.c', 25) - expected = '0123456789...5678901234.c' + test = fixed_width_file_name("0123456789012345678901234.c", 25) + expected = "0123456789...5678901234.c" assert test == expected def test_fixed_width_file_name_with_file_name_smaller_than_max_length_is_not_shortened(self): - file_name = '0123456789012345678901234.c' + file_name = "0123456789012345678901234.c" test = fixed_width_file_name(file_name, max_length=50) assert test == file_name def test_fixed_width_file_name_with_file_name_at_max_length_is_not_shortened(self): - test = fixed_width_file_name('01234567890123456789012.c', 25) - expected = '01234567890123456789012.c' + test = fixed_width_file_name("01234567890123456789012.c", 25) + expected = "01234567890123456789012.c" assert test == expected def test_fixed_width_file_name_with_file_name_smaller_than_max_length_not_shortened(self): - test = fixed_width_file_name('0123456789012345678901.c', 25) - expected = '0123456789012345678901.c' + test = fixed_width_file_name("0123456789012345678901.c", 25) + expected = "0123456789012345678901.c" assert test == expected def test_fixed_width_file_name_with_none_filename_return_empty_string(self): test = fixed_width_file_name(None, 25) - expected = '' + expected = "" assert test == expected def test_fixed_width_file_name_without_extension(self): - test = fixed_width_file_name('012345678901234567890123456', 25) - expected = '01234567890...67890123456' + test = fixed_width_file_name("012345678901234567890123456", 25) + expected = "01234567890...67890123456" assert test == expected def test_fixed_width_file_name_with_posix_path_without_shortening(self): test = fixed_width_file_name( - 'C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/drupal.js', 25) - expected = 'drupal.js' + "C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/drupal.js", 25 + ) + expected = "drupal.js" assert test == expected def test_fixed_width_file_name_with_posix_path_with_shortening(self): test = fixed_width_file_name( - 'C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/012345678901234567890123.c', 25) - expected = '0123456789...4567890123.c' + "C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/012345678901234567890123.c", + 25, + ) + expected = "0123456789...4567890123.c" assert test == expected def test_fixed_width_file_name_with_win_path_without_shortening(self): test = fixed_width_file_name( - 'C\\:Documents_and_Settings\\Boki\\Desktop\\head\\patches\\drupal6\\drupal.js', 25) - expected = 'drupal.js' + "C\\:Documents_and_Settings\\Boki\\Desktop\\head\\patches\\drupal6\\drupal.js", 25 + ) + expected = "drupal.js" assert test == expected def test_fixed_width_file_name_with_win_path_with_shortening(self): test = fixed_width_file_name( - 'C\\:Documents_and_Settings\\Boki\\Desktop\\head\\patches\\drupal6\\012345678901234567890123.c', 25) - expected = '0123456789...4567890123.c' + "C\\:Documents_and_Settings\\Boki\\Desktop\\head\\patches\\drupal6\\012345678901234567890123.c", + 25, + ) + expected = "0123456789...4567890123.c" assert test == expected def test_fixed_width_file_name_with_very_small_file_name_and_long_extension(self): - test = fixed_width_file_name('abc.abcdef', 5) + test = fixed_width_file_name("abc.abcdef", 5) # FIXME: what is expected is TBD - expected = '' + expected = "" assert test == expected class TestGroupedHelpCommand(FileDrivenTesting): - - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_GroupedHelpCommand_help_group_and_sort_order_without_custom_class(self): - - @click.command(name='scan', cls=GroupedHelpCommand) - @click.option('--opt', is_flag=True, help='Help text for option') + @click.command(name="scan", cls=GroupedHelpCommand) + @click.option("--opt", is_flag=True, help="Help text for option") def scan(opt): pass runner = CliRunner() - result = runner.invoke(scan, ['--help']) + result = runner.invoke(scan, ["--help"]) from commoncode.cliutils import MISC_GROUP + assert MISC_GROUP in result.output - assert '--opt Help text for option' in result.output + assert "--opt Help text for option" in result.output def test_GroupedHelpCommand_with_help_group_and_sort_order_with_custom_class(self): - - @click.command(name='scan', cls=GroupedHelpCommand) - @click.option('--opt', is_flag=True, sort_order=10, - help='Help text for option', cls=PluggableCommandLineOption) + @click.command(name="scan", cls=GroupedHelpCommand) + @click.option( + "--opt", + is_flag=True, + sort_order=10, + help="Help text for option", + cls=PluggableCommandLineOption, + ) def scan(opt): pass runner = CliRunner() - result = runner.invoke(scan, ['--help']) + result = runner.invoke(scan, ["--help"]) from commoncode.cliutils import MISC_GROUP - assert MISC_GROUP + ':\n --opt Help text for option\n' in result.output + + assert MISC_GROUP + ":\n --opt Help text for option\n" in result.output def test_GroupedHelpCommand_help_with_group(self): from commoncode.cliutils import CORE_GROUP - @click.command(name='scan', cls=GroupedHelpCommand) + @click.command(name="scan", cls=GroupedHelpCommand) @click.option( - '--opt', + "--opt", is_flag=True, - help='Help text for option', + help="Help text for option", help_group=CORE_GROUP, cls=PluggableCommandLineOption, ) @@ -125,5 +136,5 @@ def scan(opt): pass runner = CliRunner() - result = runner.invoke(scan, ['--help']) - assert CORE_GROUP + ':\n --opt Help text for option\n' in result.output + result = runner.invoke(scan, ["--help"]) + assert CORE_GROUP + ":\n --opt Help text for option\n" in result.output diff --git a/tests/test_cliutils_progressbar.py b/tests/test_cliutils_progressbar.py index 1ee8dca9..2683e9c4 100644 --- a/tests/test_cliutils_progressbar.py +++ b/tests/test_cliutils_progressbar.py @@ -8,13 +8,12 @@ import os -from commoncode.testcase import FileDrivenTesting from commoncode.cliutils import progressmanager +from commoncode.testcase import FileDrivenTesting class TestProgressBar(FileDrivenTesting): - - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_progressmanager_works(self): iterable = range(10) diff --git a/tests/test_codec.py b/tests/test_codec.py index e852e707..1109acfd 100644 --- a/tests/test_codec.py +++ b/tests/test_codec.py @@ -14,20 +14,19 @@ class TestCodec(TestCase): - def test_bin_to_num_basic(self): expected = 123 - result = bin_to_num(b'{') + result = bin_to_num(b"{") assert result == expected def test_bin_to_num_zero(self): expected = 0 - result = bin_to_num(b'\x00') + result = bin_to_num(b"\x00") assert result == expected def test_bin_to_num_large_number(self): expected = 432346237462348763 - result = bin_to_num(b'\x06\x00\x00\x9c\xbf\xeb\x83\xdb') + result = bin_to_num(b"\x06\x00\x00\x9c\xbf\xeb\x83\xdb") assert result == expected def test_bin_to_num_and_num_to_bin_is_idempotent(self): @@ -36,43 +35,43 @@ def test_bin_to_num_and_num_to_bin_is_idempotent(self): assert result == expected def test_num_to_bin_basic(self): - expected = b'{' + expected = b"{" result = num_to_bin(123) assert result == expected def test_num_to_bin_zero(self): - expected = b'\x00' + expected = b"\x00" result = num_to_bin(0) assert result == expected def test_num_to_bin_large_number(self): - expected = b'\x06\x00\x00\x9c\xbf\xeb\x83\xdb' + expected = b"\x06\x00\x00\x9c\xbf\xeb\x83\xdb" result = num_to_bin(432346237462348763) assert result == expected def test_num_to_bin_bin_to_num_is_idempotent(self): - expected = b'\x06\x00\x00\x9c\xbf\xeb\x83\xdb' - result = num_to_bin(bin_to_num(b'\x06\x00\x00\x9c\xbf\xeb\x83\xdb')) + expected = b"\x06\x00\x00\x9c\xbf\xeb\x83\xdb" + result = num_to_bin(bin_to_num(b"\x06\x00\x00\x9c\xbf\xeb\x83\xdb")) assert result == expected def test_urlsafe_b64encode_int_zero(self): - assert urlsafe_b64encode_int(0) == b'AA==' + assert urlsafe_b64encode_int(0) == b"AA==" def test_urlsafe_b64encode_int_basic(self): - assert urlsafe_b64encode_int(123123123123) == b'HKq1w7M=' + assert urlsafe_b64encode_int(123123123123) == b"HKq1w7M=" def test_urlsafe_b64encode_int_limit_8bits_255(self): - assert urlsafe_b64encode_int(255) == b'_w==' + assert urlsafe_b64encode_int(255) == b"_w==" def test_urlsafe_b64encode_int_limit_8bits_256(self): - assert urlsafe_b64encode_int(256) == b'AQA=' + assert urlsafe_b64encode_int(256) == b"AQA=" def test_urlsafe_b64encode_int_adds_no_padding_for_number_that_are_multiple_of_6_bits(self): - assert urlsafe_b64encode_int(0xFFFFFFFFFFFFFFFFFF) == b'____________' + assert urlsafe_b64encode_int(0xFFFFFFFFFFFFFFFFFF) == b"____________" assert len(urlsafe_b64encode_int(0xFFFFFFFFFFFF)) == 8 def test_urlsafe_b64encode_int_very_large_number(self): - b64 = (b'QAAAAAAgAAAAAQAACAAAAAAAAAAAAAAkAAIAAAAAAAAAAAAAAACAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAiAAAAAAAIAAAAAAAAAAAAAAEAACAAAAAAAA=') + b64 = b"QAAAAAAgAAAAAQAACAAAAAAAAAAAAAAkAAIAAAAAAAAAAAAAAACAAIAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAAAAAAAAAAAAAAAAAAiAAAAAAAIAAAAAAAAAAAAAAEAACAAAAAAAA=" expected = b64 num = 2678771517966886466622496485850735537232223496190189203248435106535830319026141316924949516664780383591425235756710588949364368366679435700855700642969357960349427980681242720502045830438444033569999428606714388704082526548154984676817460705606960919023941301616034362869262429593297635158449513824256 result = urlsafe_b64encode_int(num) diff --git a/tests/test_command.py b/tests/test_command.py index 1f54c73b..173b6c61 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -9,150 +9,174 @@ import os import sys - from unittest.case import skipIf from commoncode import command -from commoncode.testcase import FileBasedTesting from commoncode.system import on_linux from commoncode.system import on_mac from commoncode.system import on_windows from commoncode.system import py36 +from commoncode.testcase import FileBasedTesting class TestCommand(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") - @skipIf(py36, 'This fails on Python 3.6 https://bugs.python.org/issue26919') + @skipIf(py36, "This fails on Python 3.6 https://bugs.python.org/issue26919") def test_execute_can_handle_non_ascii_output(self): # Popen returns a *binary* string with non-ascii chars: skips these python = sys.executable rc, stdout, stderr = command.execute( - python, ['-c', 'print("non ascii: été just passed it !")'] + python, ["-c", 'print("non ascii: été just passed it !")'] ) - assert stderr == '' - assert stdout == 'non ascii: ete just passed it !' + assert stderr == "" + assert stdout == "non ascii: ete just passed it !" assert rc == 0 # do not throw exception - stdout.encode('ascii') + stdout.encode("ascii") def test_execute_(self): python = sys.executable - rc, stdout, stderr = command.execute( - python, ['-c', 'print("foobar")'] - ) - assert stderr == '' - assert stdout == 'foobar' + rc, stdout, stderr = command.execute(python, ["-c", 'print("foobar")']) + assert stderr == "" + assert stdout == "foobar" assert rc == 0 # do not throw exception - stdout.encode('ascii') + stdout.encode("ascii") def test_execute2(self): python = sys.executable - rc, stdout, stderr = command.execute2( - python, ['-c', 'print("foobar")'] - ) - assert stderr == '' - assert stdout == 'foobar' + rc, stdout, stderr = command.execute2(python, ["-c", 'print("foobar")']) + assert stderr == "" + assert stdout == "foobar" assert rc == 0 # do not throw exception - stdout.encode('ascii') + stdout.encode("ascii") - @skipIf(not on_linux, 'Linux only') + @skipIf(not on_linux, "Linux only") def test_update_path_var_on_linux(self): - existing_path_var = '/usr/bin:/usr/local' + existing_path_var = "/usr/bin:/usr/local" - new_path = b'foo\xb1bar' + new_path = b"foo\xb1bar" updated_path = command.update_path_var(existing_path_var, new_path) - assert updated_path == 'foo\udcb1bar:/usr/bin:/usr/local' + assert updated_path == "foo\udcb1bar:/usr/bin:/usr/local" - new_path = u'/bin/foo\udcb1bar' + new_path = "/bin/foo\udcb1bar" updated_path = command.update_path_var(updated_path, new_path) - assert updated_path == '/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local' + assert updated_path == "/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local" - new_path = b'/bin/foo\xb1bar' + new_path = b"/bin/foo\xb1bar" updated_path = command.update_path_var(updated_path, new_path) - assert updated_path == '/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local' + assert updated_path == "/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local" - new_path = u'foo\udcb1bar' + new_path = "foo\udcb1bar" updated_path = command.update_path_var(updated_path, new_path) - assert updated_path == '/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local' + assert updated_path == "/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local" - new_path = b'foo\xb1bar' + new_path = b"foo\xb1bar" updated_path = command.update_path_var(updated_path, new_path) - assert updated_path == '/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local' + assert updated_path == "/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local" - @skipIf(not on_mac, 'Mac only') + @skipIf(not on_mac, "Mac only") def test_update_path_var_on_mac(self): - existing_path_var = '/usr/bin:/usr/local' + existing_path_var = "/usr/bin:/usr/local" - new_path = u'foo\udcb1bar' + new_path = "foo\udcb1bar" updated_path = command.update_path_var(existing_path_var, new_path) - assert updated_path == 'foo\udcb1bar:/usr/bin:/usr/local' + assert updated_path == "foo\udcb1bar:/usr/bin:/usr/local" - new_path = b'/bin/foo\xb1bar' + new_path = b"/bin/foo\xb1bar" updated_path = command.update_path_var(updated_path, new_path) - assert updated_path == '/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local' + assert updated_path == "/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local" - new_path = u'foo\udcb1bar' + new_path = "foo\udcb1bar" updated_path = command.update_path_var(updated_path, new_path) - assert updated_path == '/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local' + assert updated_path == "/bin/foo\udcb1bar:foo\udcb1bar:/usr/bin:/usr/local" - @skipIf(not on_windows, 'Windows only') + @skipIf(not on_windows, "Windows only") def test_update_path_var_on_windows(self): - existing_path_var = u'c:\\windows;C:Program Files' + existing_path_var = "c:\\windows;C:Program Files" - new_path = u'foo\udcb1bar' + new_path = "foo\udcb1bar" updated_path = command.update_path_var(existing_path_var, new_path) - assert updated_path == u'foo\udcb1bar;c:\\windows;C:Program Files' + assert updated_path == "foo\udcb1bar;c:\\windows;C:Program Files" - new_path = u'foo\udcb1bar' + new_path = "foo\udcb1bar" updated_path = command.update_path_var(updated_path, new_path) - assert updated_path == u'foo\udcb1bar;c:\\windows;C:Program Files' + assert updated_path == "foo\udcb1bar;c:\\windows;C:Program Files" def test_searchable_paths(self): - d1 = self.get_temp_dir('foo') - d2 = self.get_temp_dir('bar') + d1 = self.get_temp_dir("foo") + d2 = self.get_temp_dir("bar") ps = os.pathsep - os.environ['FOO_SCANCODE_TEST1'] = f'{ps}{d1}{ps}{ps}{d2}{ps}' - os.environ['FOO_SCANCODE_TEST2'] = f'{ps}{d2}{ps}{ps}{d1}{ps}/NOTADIR' + os.environ["FOO_SCANCODE_TEST1"] = f"{ps}{d1}{ps}{ps}{d2}{ps}" + os.environ["FOO_SCANCODE_TEST2"] = f"{ps}{d2}{ps}{ps}{d1}{ps}/NOTADIR" - env_vars = 'FOO_SCANCODE_TEST1', 'FOO_SCANCODE_TEST2' + env_vars = "FOO_SCANCODE_TEST1", "FOO_SCANCODE_TEST2" expected = d1, d2, d2, d1 results = command.searchable_paths(env_vars=env_vars) if on_windows: for res, exp in zip(results, expected): - _, _, r = res.rpartition('\\') - _, _, e = exp.rpartition('\\') + _, _, r = res.rpartition("\\") + _, _, e = exp.rpartition("\\") assert r == e elif on_mac: # macOS somehow adds a /private to the paths in the CI as a side- # effect of calling "realpath" and likely resolving links - expected = f'/private{d1}', f'/private{d2}', f'/private{d2}', f'/private{d1}' + expected = f"/private{d1}", f"/private{d2}", f"/private{d2}", f"/private{d1}" assert expected == results else: assert expected == results def test_find_in_path(self): - d1 = self.get_temp_dir('foo') - d2 = self.get_temp_dir('bar') - filename = 'baz' + d1 = self.get_temp_dir("foo") + d2 = self.get_temp_dir("bar") + filename = "baz" assert None == command.find_in_path( - filename, searchable_paths=(d1, d2,)) + filename, + searchable_paths=( + d1, + d2, + ), + ) f2 = os.path.join(d2, filename) - with open(f2, 'w') as o: - o.write('some') - - assert f2 == command.find_in_path(filename, searchable_paths=(d1, d2,)) - assert f2 == command.find_in_path(filename, searchable_paths=(d2, d1,)) + with open(f2, "w") as o: + o.write("some") + + assert f2 == command.find_in_path( + filename, + searchable_paths=( + d1, + d2, + ), + ) + assert f2 == command.find_in_path( + filename, + searchable_paths=( + d2, + d1, + ), + ) f1 = os.path.join(d1, filename) - with open(f1, 'w') as o: - o.write('some') - - assert f1 == command.find_in_path(filename, searchable_paths=(d1, d2,)) - assert f2 == command.find_in_path(filename, searchable_paths=(d2, d1,)) + with open(f1, "w") as o: + o.write("some") + + assert f1 == command.find_in_path( + filename, + searchable_paths=( + d1, + d2, + ), + ) + assert f2 == command.find_in_path( + filename, + searchable_paths=( + d2, + d1, + ), + ) diff --git a/tests/test_date.py b/tests/test_date.py index 4f30f27a..05a29663 100644 --- a/tests/test_date.py +++ b/tests/test_date.py @@ -7,19 +7,16 @@ # import os - from datetime import datetime import commoncode.date - from commoncode import testcase class TestDate(testcase.FileBasedTesting): - def test_secs_from_epoch_can_handle_micro_and_nano_secs(self): test_file = self.get_temp_file() - open(test_file, 'w').close() + open(test_file, "w").close() # setting modified time to desired values os.utime(test_file, (1301420665.046481, 1301420665.046481)) # otherwise the issue does not happen (ie. on mac) @@ -29,19 +26,19 @@ def test_secs_from_epoch_can_handle_micro_and_nano_secs(self): def test_get_file_mtime_for_a_new_file(self): test_file = self.get_temp_file() - open(test_file, 'w').close() + open(test_file, "w").close() def as_yyyymmdd(s): return s[:10] - now = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ') + now = datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") result = commoncode.date.get_file_mtime(test_file) assert as_yyyymmdd(result) == as_yyyymmdd(now) def test_get_file_mtime_for_a_modified_file(self): test_file = self.get_temp_file() - open(test_file, 'w').close() - expected = u'1992-05-09 00:00:00' + open(test_file, "w").close() + expected = "1992-05-09 00:00:00" m_ts = (24 * 3600) * 134 + (24 * 3600 * 365) * 22 # setting modified time to expected values os.utime(test_file, (m_ts, m_ts)) @@ -49,8 +46,8 @@ def test_get_file_mtime_for_a_modified_file(self): def test_get_file_mtime_for_a_modified_file_2(self): test_file = self.get_temp_file() - open(test_file, 'w').close() + open(test_file, "w").close() # setting modified time to expected values - expected = u'2011-01-06 14:35:00' + expected = "2011-01-06 14:35:00" os.utime(test_file, (1294324500, 1294324500)) assert commoncode.date.get_file_mtime(test_file) == expected diff --git a/tests/test_distro.py b/tests/test_distro.py index c4b5d5a7..48521c1b 100644 --- a/tests/test_distro.py +++ b/tests/test_distro.py @@ -7,14 +7,14 @@ class TestDistro(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_parse_os_release(self): - test_dir = self.get_test_loc('distro/os-release') + test_dir = self.get_test_loc("distro/os-release") for test_file in resource_iter(test_dir, with_dirs=False): - if test_file.endswith('expected.json'): + if test_file.endswith("expected.json"): continue - expected = test_file + '-expected.json' + expected = test_file + "-expected.json" result = distro.parse_os_release(test_file) check_against_expected_json_file(result, expected, regen=False) diff --git a/tests/test_fileset.py b/tests/test_fileset.py index 79e44a8f..2c6e5ef7 100644 --- a/tests/test_fileset.py +++ b/tests/test_fileset.py @@ -13,82 +13,84 @@ class FilesetTest(commoncode.testcase.FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_load(self): - irf = self.get_test_loc('fileset/scancodeignore.lst') + irf = self.get_test_loc("fileset/scancodeignore.lst") result = fileset.load(irf) - assert result == ['/foo/*', '!/foobar/*', 'bar/*', '#comment'] + assert result == ["/foo/*", "!/foobar/*", "bar/*", "#comment"] def test_is_included_basic(self): - assert fileset.is_included('/common/src/', {}, {}) - assert fileset.is_included('/common/src/', None, None) + assert fileset.is_included("/common/src/", {}, {}) + assert fileset.is_included("/common/src/", None, None) assert not fileset.is_included(None, None, None) def test_is_included_in_fileset(self): - incs = {'/common/src/*': '.scanignore'} - excs = {'/common/src/*.so': '.scanignore'} + incs = {"/common/src/*": ".scanignore"} + excs = {"/common/src/*.so": ".scanignore"} assert not fileset.is_included(None, incs, excs) - assert not fileset.is_included('', incs, excs) - assert not fileset.is_included('/', incs, excs) - assert fileset.is_included('/common/src/', incs, excs) - assert not fileset.is_included('/common/bin/', incs, excs) + assert not fileset.is_included("", incs, excs) + assert not fileset.is_included("/", incs, excs) + assert fileset.is_included("/common/src/", incs, excs) + assert not fileset.is_included("/common/bin/", incs, excs) def test_is_included_in_fileset_2(self): - incs = {'src*': '.scanignore'} - excs = {'src/ab': '.scanignore'} + incs = {"src*": ".scanignore"} + excs = {"src/ab": ".scanignore"} assert not fileset.is_included(None, incs, excs) - assert not fileset.is_included('', incs, excs) - assert not fileset.is_included('/', incs, excs) - assert fileset.is_included('/common/src/', incs, excs) - assert not fileset.is_included('src/ab', incs, excs) - assert fileset.is_included('src/abbab', incs, excs) + assert not fileset.is_included("", incs, excs) + assert not fileset.is_included("/", incs, excs) + assert fileset.is_included("/common/src/", incs, excs) + assert not fileset.is_included("src/ab", incs, excs) + assert fileset.is_included("src/abbab", incs, excs) def test_is_included_is_included_exclusions(self): - incs = {'/src/*': '.scanignore'} - excs = {'/src/*.so': '.scanignore'} - assert not fileset.is_included('/src/dist/build/mylib.so', incs, excs) + incs = {"/src/*": ".scanignore"} + excs = {"/src/*.so": ".scanignore"} + assert not fileset.is_included("/src/dist/build/mylib.so", incs, excs) def test_is_included_is_included_exclusions_2(self): - incs = {'src': '.scanignore'} - excs = {'src/*.so': '.scanignore'} - assert fileset.is_included('/some/src/this/that', incs, excs) - assert not fileset.is_included('/src/dist/build/mylib.so', incs, excs) + incs = {"src": ".scanignore"} + excs = {"src/*.so": ".scanignore"} + assert fileset.is_included("/some/src/this/that", incs, excs) + assert not fileset.is_included("/src/dist/build/mylib.so", incs, excs) def test_is_included_empty_exclusions(self): - incs = {'/src/*': '.scanignore'} - excs = {'': '.scanignore'} - assert fileset.is_included('/src/dist/build/mylib.so', incs, excs) + incs = {"/src/*": ".scanignore"} + excs = {"": ".scanignore"} + assert fileset.is_included("/src/dist/build/mylib.so", incs, excs) def test_is_included_sources(self): - incs = {'/home/elf/elf-0.5/*': '.scanignore'} - excs = {'/home/elf/elf-0.5/src/elf': '.scanignore', - '/home/elf/elf-0.5/src/elf.o': '.scanignore'} - assert not fileset.is_included('/home/elf/elf-0.5/src/elf', incs, excs) + incs = {"/home/elf/elf-0.5/*": ".scanignore"} + excs = { + "/home/elf/elf-0.5/src/elf": ".scanignore", + "/home/elf/elf-0.5/src/elf.o": ".scanignore", + } + assert not fileset.is_included("/home/elf/elf-0.5/src/elf", incs, excs) def test_is_included_dot_svn(self): - incs = {'*/.svn/*': '.scanignore'} + incs = {"*/.svn/*": ".scanignore"} excs = {} - assert fileset.is_included('home/common/tools/elf/.svn/', incs, excs) - assert fileset.is_included('home/common/tools/.svn/this', incs, excs) - assert not fileset.is_included('home/common/tools/this', incs, excs) + assert fileset.is_included("home/common/tools/elf/.svn/", incs, excs) + assert fileset.is_included("home/common/tools/.svn/this", incs, excs) + assert not fileset.is_included("home/common/tools/this", incs, excs) def test_is_included_dot_svn_with_excludes(self): - incs = {'*/.svn/*': '.scanignore'} - excs = {'*/.git/*': '.scanignore'} - assert fileset.is_included('home/common/tools/elf/.svn/', incs, excs) - assert fileset.is_included('home/common/tools/.svn/this', incs, excs) - assert not fileset.is_included('home/common/.git/this', incs, excs) + incs = {"*/.svn/*": ".scanignore"} + excs = {"*/.git/*": ".scanignore"} + assert fileset.is_included("home/common/tools/elf/.svn/", incs, excs) + assert fileset.is_included("home/common/tools/.svn/this", incs, excs) + assert not fileset.is_included("home/common/.git/this", incs, excs) def test_get_matches(self): - patterns = {'*/.svn/*': '.scanignore'} - assert fileset.get_matches('home/common/tools/elf/.svn/', patterns) - assert fileset.get_matches('home/common/tools/.svn/this', patterns) - assert not fileset.get_matches('home/common/.git/this', patterns) + patterns = {"*/.svn/*": ".scanignore"} + assert fileset.get_matches("home/common/tools/elf/.svn/", patterns) + assert fileset.get_matches("home/common/tools/.svn/this", patterns) + assert not fileset.get_matches("home/common/.git/this", patterns) def test_get_matches_accepts_a_list_or_tuple(self): - patterns = ['*/.svn/*'] - assert fileset.get_matches('home/common/tools/elf/.svn/', patterns) + patterns = ["*/.svn/*"] + assert fileset.get_matches("home/common/tools/elf/.svn/", patterns) - patterns = '*/.svn/*', - assert fileset.get_matches('home/common/tools/elf/.svn/', patterns) + patterns = ("*/.svn/*",) + assert fileset.get_matches("home/common/tools/elf/.svn/", patterns) diff --git a/tests/test_filetype.py b/tests/test_filetype.py index 378313a0..5a8a14e1 100644 --- a/tests/test_filetype.py +++ b/tests/test_filetype.py @@ -7,35 +7,34 @@ # import os -from os.path import join from os.path import exists +from os.path import join from unittest import skipIf +import commoncode.testcase from commoncode import filetype from commoncode import fileutils from commoncode.system import on_posix from commoncode.system import on_windows from commoncode.system import py3 - -import commoncode.testcase from commoncode.testcase import FileBasedTesting from commoncode.testcase import make_non_readable from commoncode.testcase import make_non_writable class TypeTest(commoncode.testcase.FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_get_size_on_file(self): - test_file = self.get_test_loc('filetype/size/Image1.eps') + test_file = self.get_test_loc("filetype/size/Image1.eps") assert filetype.get_size(test_file) == 12388 def test_get_size_on_directory(self): - test_dir = self.get_test_loc('filetype/size', copy=True) + test_dir = self.get_test_loc("filetype/size", copy=True) assert filetype.get_size(test_dir) == 12400 def test_get_type(self): - test_dir = self.extract_test_tar('filetype/types.tar', verbatim=True) + test_dir = self.extract_test_tar("filetype/types.tar", verbatim=True) results = [] for root, dirs, files in os.walk(test_dir): for d in dirs: @@ -44,27 +43,33 @@ def test_get_type(self): results.append((f, filetype.get_type(os.path.join(root, f)))) expected = [ - ('5-DIRTYPE', 'd'), - ('0-REGTYPE', 'f'), - ('0-REGTYPE-TEXT', 'f'), - ('0-REGTYPE-VEEEERY_LONG_NAME___________________________________' - '______________________________________________________________' - '____________________155', 'f'), - ('1-LNKTYPE', 'f'), - ('S-SPARSE', 'f'), - ('S-SPARSE-WITH-NULLS', 'f') + ("5-DIRTYPE", "d"), + ("0-REGTYPE", "f"), + ("0-REGTYPE-TEXT", "f"), + ( + "0-REGTYPE-VEEEERY_LONG_NAME___________________________________" + "______________________________________________________________" + "____________________155", + "f", + ), + ("1-LNKTYPE", "f"), + ("S-SPARSE", "f"), + ("S-SPARSE-WITH-NULLS", "f"), ] # symlinks and special files are not supported on win if on_posix: - expected += [('2-SYMTYPE', 'l'), ('6-FIFOTYPE', 's'), ] + expected += [ + ("2-SYMTYPE", "l"), + ("6-FIFOTYPE", "s"), + ] try: assert sorted(results) == sorted(expected) except Exception as e: if on_windows and py3: # On some Windows symlinkes are detected OK (Windows 10?) but not in Windows 7 - expected += [('2-SYMTYPE', 'l')] + expected += [("2-SYMTYPE", "l")] assert sorted(results) == sorted(expected) else: raise e @@ -75,8 +80,8 @@ def test_is_rwx_with_none(self): assert not filetype.is_executable(None) def test_is_readable_is_writeable_file(self): - base_dir = self.get_test_loc('filetype/readwrite', copy=True) - test_file = os.path.join(os.path.join(base_dir, 'sub'), 'file') + base_dir = self.get_test_loc("filetype/readwrite", copy=True) + test_file = os.path.join(os.path.join(base_dir, "sub"), "file") try: assert filetype.is_readable(test_file) @@ -92,8 +97,8 @@ def test_is_readable_is_writeable_file(self): fileutils.chmod(base_dir, fileutils.RW, recurse=True) def test_is_readable_is_writeable_dir(self): - base_dir = self.get_test_loc('filetype/readwrite', copy=True) - test_dir = os.path.join(base_dir, 'sub') + base_dir = self.get_test_loc("filetype/readwrite", copy=True) + test_dir = os.path.join(base_dir, "sub") try: assert filetype.is_readable(test_dir) @@ -117,11 +122,11 @@ def test_is_readable_is_writeable_dir(self): class CountTest(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def get_test_count_dir(self): - test_dir = self.get_test_loc('count/filecount', copy=True) - sub3 = join(test_dir, 'dir', 'sub3') + test_dir = self.get_test_loc("count/filecount", copy=True) + sub3 = join(test_dir, "dir", "sub3") if not exists(sub3): os.makedirs(sub3) return test_dir @@ -132,8 +137,8 @@ def test_get_file_count_with_empty_dir(self): def test_get_file_count_with_single_file(self): test_file = self.get_temp_file() - with open(test_file, 'w') as f: - f.write(u'') + with open(test_file, "w") as f: + f.write("") assert filetype.is_file(test_file) assert filetype.get_file_count(test_file) == 1 @@ -150,21 +155,21 @@ def test_get_file_size_and_count(self): def test_get_file_size(self): test_dir = self.get_test_count_dir() tests = ( - ('dir/a.txt', 2), - ('dir/b.txt', 2), - ('dir/c.txt', 2), - ('dir/sub1/a.txt', 2), - ('dir/sub1/b.txt', 2), - ('dir/sub1/c.txt', 2), - ('dir/sub1/subsub/a.txt', 2), - ('dir/sub1/subsub/b.txt', 2), - ('dir/sub1/subsub', 4), - ('dir/sub1', 10), - ('dir/sub2/a.txt', 2), - ('dir/sub2', 2), - ('dir/sub3', 0), - ('dir/', 18), - ('', 18), + ("dir/a.txt", 2), + ("dir/b.txt", 2), + ("dir/c.txt", 2), + ("dir/sub1/a.txt", 2), + ("dir/sub1/b.txt", 2), + ("dir/sub1/c.txt", 2), + ("dir/sub1/subsub/a.txt", 2), + ("dir/sub1/subsub/b.txt", 2), + ("dir/sub1/subsub", 4), + ("dir/sub1", 10), + ("dir/sub2/a.txt", 2), + ("dir/sub2", 2), + ("dir/sub3", 0), + ("dir/", 18), + ("", 18), ) for test_file, size in tests: result = filetype.get_size(os.path.join(test_dir, test_file)) @@ -173,21 +178,21 @@ def test_get_file_size(self): def test_get_file_count(self): test_dir = self.get_test_count_dir() tests = ( - ('dir/a.txt', 1), - ('dir/b.txt', 1), - ('dir/c.txt', 1), - ('dir/sub1/a.txt', 1), - ('dir/sub1/b.txt', 1), - ('dir/sub1/c.txt', 1), - ('dir/sub1/subsub/a.txt', 1), - ('dir/sub1/subsub/b.txt', 1), - ('dir/sub1/subsub', 2), - ('dir/sub1', 5), - ('dir/sub2/a.txt', 1), - ('dir/sub2', 1), - ('dir/sub3', 0), - ('dir/', 9), - ('', 9), + ("dir/a.txt", 1), + ("dir/b.txt", 1), + ("dir/c.txt", 1), + ("dir/sub1/a.txt", 1), + ("dir/sub1/b.txt", 1), + ("dir/sub1/c.txt", 1), + ("dir/sub1/subsub/a.txt", 1), + ("dir/sub1/subsub/b.txt", 1), + ("dir/sub1/subsub", 2), + ("dir/sub1", 5), + ("dir/sub2/a.txt", 1), + ("dir/sub2", 1), + ("dir/sub3", 0), + ("dir/", 9), + ("", 9), ) for test_file, count in tests: result = filetype.get_file_count(os.path.join(test_dir, test_file)) @@ -195,22 +200,22 @@ def test_get_file_count(self): class SymlinkTest(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") - @skipIf(on_windows, 'os.symlink does not work on Windows') + @skipIf(on_windows, "os.symlink does not work on Windows") def test_is_file(self): - test_file = self.get_test_loc('symlink/test', copy=True) + test_file = self.get_test_loc("symlink/test", copy=True) temp_dir = fileutils.get_temp_dir() - test_link = join(temp_dir, 'test-link') + test_link = join(temp_dir, "test-link") os.symlink(test_file, test_link) assert filetype.is_file(test_link, follow_symlinks=True) assert not filetype.is_file(test_link, follow_symlinks=False) - @skipIf(on_windows, 'os.symlink does not work on Windows') + @skipIf(on_windows, "os.symlink does not work on Windows") def test_is_dir(self): - test_dir = self.get_test_loc('symlink', copy=True) + test_dir = self.get_test_loc("symlink", copy=True) temp_dir = fileutils.get_temp_dir() - test_link = join(temp_dir, 'test-dir-link') + test_link = join(temp_dir, "test-dir-link") os.symlink(test_dir, test_link) assert filetype.is_dir(test_link, follow_symlinks=True) assert not filetype.is_dir(test_link, follow_symlinks=False) diff --git a/tests/test_fileutils.py b/tests/test_fileutils.py index 049c1c71..73cedc1e 100644 --- a/tests/test_fileutils.py +++ b/tests/test_fileutils.py @@ -16,9 +16,9 @@ from commoncode import fileutils from commoncode.fileutils import as_posixpath from commoncode.system import on_linux -from commoncode.system import on_posix from commoncode.system import on_mac from commoncode.system import on_macos_14_or_higher +from commoncode.system import on_posix from commoncode.system import on_windows from commoncode.testcase import FileBasedTesting from commoncode.testcase import make_non_executable @@ -26,17 +26,18 @@ from commoncode.testcase import make_non_writable -@skip('Somehow permissions tests do not work OK yet on Python 3') +@skip("Somehow permissions tests do not work OK yet on Python 3") class TestPermissionsDeletions(FileBasedTesting): """ This is failing for now on Python 3 """ - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_delete_unwritable_directory_and_files(self): - base_dir = self.get_test_loc('fileutils/readwrite', copy=True) - test_dir = join(base_dir, 'sub') - test_file = join(test_dir, 'file') + base_dir = self.get_test_loc("fileutils/readwrite", copy=True) + test_dir = join(base_dir, "sub") + test_file = join(test_dir, "file") try: # note: there are no unread/writable dir on windows @@ -59,16 +60,17 @@ class TestPermissions(FileBasedTesting): Several assertions or test are skipped on non posix OSes. Windows handles permissions and special files differently. """ - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_chmod_on_non_existing_file_throws_no_exception(self): - fileutils.chmod('some non existing dir', fileutils.RWX) + fileutils.chmod("some non existing dir", fileutils.RWX) def test_chmod_read_write_recursively_on_dir(self): - test_dir = self.get_test_loc('fileutils/executable', copy=True) - test_file = join(test_dir, 'deep1', 'deep2', 'ctags') - test_dir2 = join(test_dir, 'deep1', 'deep2') - parent = join(test_dir, 'deep1') + test_dir = self.get_test_loc("fileutils/executable", copy=True) + test_file = join(test_dir, "deep1", "deep2", "ctags") + test_dir2 = join(test_dir, "deep1", "deep2") + parent = join(test_dir, "deep1") try: make_non_writable(test_file) @@ -94,10 +96,10 @@ def test_chmod_read_write_recursively_on_dir(self): fileutils.chmod(test_dir, fileutils.RW, recurse=True) def test_chmod_read_write_non_recursively_on_dir(self): - test_dir = self.get_test_loc('fileutils/executable', copy=True) - test_file = join(test_dir, 'deep1', 'deep2', 'ctags') - test_dir = join(test_dir, 'deep1', 'deep2') - parent = join(test_dir, 'deep1') + test_dir = self.get_test_loc("fileutils/executable", copy=True) + test_file = join(test_dir, "deep1", "deep2", "ctags") + test_dir = join(test_dir, "deep1", "deep2") + parent = join(test_dir, "deep1") try: # setup @@ -124,8 +126,8 @@ def test_chmod_read_write_non_recursively_on_dir(self): fileutils.chmod(test_dir, fileutils.RW, recurse=True) def test_chmod_read_write_file(self): - test_dir = self.get_test_loc('fileutils/executable', copy=True) - test_file = join(test_dir, 'deep1', 'deep2', 'ctags') + test_dir = self.get_test_loc("fileutils/executable", copy=True) + test_file = join(test_dir, "deep1", "deep2", "ctags") try: make_non_writable(test_file) @@ -138,8 +140,8 @@ def test_chmod_read_write_file(self): fileutils.chmod(test_dir, fileutils.RW, recurse=True) def test_chmod_read_write_exec_dir(self): - test_dir = self.get_test_loc('fileutils/executable', copy=True) - test_file = join(test_dir, 'deep1', 'deep2', 'ctags') + test_dir = self.get_test_loc("fileutils/executable", copy=True) + test_file = join(test_dir, "deep1", "deep2", "ctags") try: if on_posix: @@ -158,8 +160,8 @@ def test_chmod_read_write_exec_dir(self): def test_copyfile_does_not_keep_permissions(self): src_file = self.get_temp_file() dest = self.get_temp_dir() - with open(src_file, 'w') as f: - f.write(u'') + with open(src_file, "w") as f: + f.write("") try: make_non_readable(src_file) if on_posix: @@ -173,28 +175,28 @@ def test_copyfile_does_not_keep_permissions(self): fileutils.chmod(dest, fileutils.RW, recurse=True) def test_copytree_does_not_keep_non_writable_permissions(self): - src = self.get_test_loc('fileutils/exec', copy=True) + src = self.get_test_loc("fileutils/exec", copy=True) dst = self.get_temp_dir() try: - src_file = join(src, 'subtxt/a.txt') + src_file = join(src, "subtxt/a.txt") make_non_writable(src_file) assert not filetype.is_writable(src_file) - src_dir = join(src, 'subtxt') + src_dir = join(src, "subtxt") make_non_writable(src_dir) if on_posix: assert not filetype.is_writable(src_dir) # copy proper - dest_dir = join(dst, 'dest') + dest_dir = join(dst, "dest") fileutils.copytree(src, dest_dir) - dst_file = join(dest_dir, 'subtxt/a.txt') + dst_file = join(dest_dir, "subtxt/a.txt") assert os.path.exists(dst_file) assert filetype.is_writable(dst_file) - dest_dir2 = join(dest_dir, 'subtxt') + dest_dir2 = join(dest_dir, "subtxt") assert os.path.exists(dest_dir2) assert filetype.is_writable(dest_dir) finally: @@ -202,10 +204,10 @@ def test_copytree_does_not_keep_non_writable_permissions(self): fileutils.chmod(dst, fileutils.RW, recurse=True) def test_copytree_copies_unreadable_files(self): - src = self.get_test_loc('fileutils/exec', copy=True) + src = self.get_test_loc("fileutils/exec", copy=True) dst = self.get_temp_dir() - src_file1 = join(src, 'a.bat') - src_file2 = join(src, 'subtxt', 'a.txt') + src_file1 = join(src, "a.bat") + src_file2 = join(src, "subtxt", "a.txt") try: # make some unreadable source files @@ -218,14 +220,14 @@ def test_copytree_copies_unreadable_files(self): assert not filetype.is_readable(src_file2) # copy proper - dest_dir = join(dst, 'dest') + dest_dir = join(dst, "dest") fileutils.copytree(src, dest_dir) - dest_file1 = join(dest_dir, 'a.bat') + dest_file1 = join(dest_dir, "a.bat") assert os.path.exists(dest_file1) assert filetype.is_readable(dest_file1) - dest_file2 = join(dest_dir, 'subtxt', 'a.txt') + dest_file2 = join(dest_dir, "subtxt", "a.txt") assert os.path.exists(dest_file2) assert filetype.is_readable(dest_file2) @@ -235,21 +237,21 @@ def test_copytree_copies_unreadable_files(self): class TestFileUtils(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") - @skipIf(on_windows, 'Windows handles special files differently.') + @skipIf(on_windows, "Windows handles special files differently.") def test_copytree_does_not_copy_fifo(self): # Windows does not support pipes - src = self.get_test_loc('fileutils/filetype', copy=True) + src = self.get_test_loc("fileutils/filetype", copy=True) dest = self.get_temp_dir() - src_file = join(src, 'myfifo') + src_file = join(src, "myfifo") os.mkfifo(src_file) # NOQA - dest_dir = join(dest, 'dest') + dest_dir = join(dest, "dest") fileutils.copytree(src, dest_dir) - assert not os.path.exists(join(dest_dir, 'myfifo')) + assert not os.path.exists(join(dest_dir, "myfifo")) def test_copyfile_keeps_modified_date(self): - test_file = self.get_test_loc('fileutils/exec/subtxt/a.txt', copy=True) + test_file = self.get_test_loc("fileutils/exec/subtxt/a.txt", copy=True) dest = self.get_temp_file() expected = 1289918700 os.utime(test_file, (expected, expected)) @@ -258,25 +260,25 @@ def test_copyfile_keeps_modified_date(self): assert result == expected def test_copyfile_can_copy_file_to_dir_keeping_full_file_name(self): - test_file = self.get_test_loc('fileutils/exec/subtxt/a.txt', copy=True) + test_file = self.get_test_loc("fileutils/exec/subtxt/a.txt", copy=True) dest = self.get_temp_dir() - expected = os.path.join(dest, 'a.txt') + expected = os.path.join(dest, "a.txt") fileutils.copyfile(test_file, dest) assert os.path.exists(expected) def test_resource_name(self): - assert fileutils.resource_name('/a/b/d/f/f') == 'f' - assert fileutils.resource_name('/a/b/d/f/f/') == 'f' - assert fileutils.resource_name('a/b/d/f/f/') == 'f' - assert fileutils.resource_name('/a/b/d/f/f.a') == 'f.a' - assert fileutils.resource_name('/a/b/d/f/f.a/') == 'f.a' - assert fileutils.resource_name('a/b/d/f/f.a') == 'f.a' - assert fileutils.resource_name('f.a') == 'f.a' - - @skipIf(on_windows, 'Windows FS encoding is ... different!') + assert fileutils.resource_name("/a/b/d/f/f") == "f" + assert fileutils.resource_name("/a/b/d/f/f/") == "f" + assert fileutils.resource_name("a/b/d/f/f/") == "f" + assert fileutils.resource_name("/a/b/d/f/f.a") == "f.a" + assert fileutils.resource_name("/a/b/d/f/f.a/") == "f.a" + assert fileutils.resource_name("a/b/d/f/f.a") == "f.a" + assert fileutils.resource_name("f.a") == "f.a" + + @skipIf(on_windows, "Windows FS encoding is ... different!") def test_fsdecode_and_fsencode_are_idempotent(self): - a = b'foo\xb1bar' - b = u'foo\udcb1bar' + a = b"foo\xb1bar" + b = "foo\udcb1bar" assert os.fsencode(os.fsdecode(a)) == a assert os.fsencode(os.fsdecode(b)) == a assert os.fsdecode(os.fsencode(a)) == b @@ -284,62 +286,66 @@ def test_fsdecode_and_fsencode_are_idempotent(self): class TestFileUtilsWalk(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_os_walk_with_unicode_path(self): - test_dir = self.extract_test_zip('fileutils/walk/unicode.zip') - test_dir = join(test_dir, 'unicode') + test_dir = self.extract_test_zip("fileutils/walk/unicode.zip") + test_dir = join(test_dir, "unicode") test_dir = str(test_dir) result = list(os.walk(test_dir)) expected = [ - (str(test_dir), ['a'], [u'2.csv']), - (str(test_dir) + sep + 'a', [], [u'gru\u0308n.png']) + (str(test_dir), ["a"], ["2.csv"]), + (str(test_dir) + sep + "a", [], ["gru\u0308n.png"]), ] assert result == expected def test_fileutils_walk(self): - test_dir = self.get_test_loc('fileutils/walk') - base = self.get_test_loc('fileutils') - result = [(as_posixpath(t.replace(base, '')), d, sorted(f),) - for t, d, f in fileutils.walk(test_dir)] + test_dir = self.get_test_loc("fileutils/walk") + base = self.get_test_loc("fileutils") + result = [ + ( + as_posixpath(t.replace(base, "")), + d, + sorted(f), + ) + for t, d, f in fileutils.walk(test_dir) + ] expected = [ - ('/walk', ['d1'], ['f', 'unicode.zip']), - ('/walk/d1', ['d2'], ['f1']), - ('/walk/d1/d2', ['d3'], ['f2']), - ('/walk/d1/d2/d3', [], ['f3']) + ("/walk", ["d1"], ["f", "unicode.zip"]), + ("/walk/d1", ["d2"], ["f1"]), + ("/walk/d1/d2", ["d3"], ["f2"]), + ("/walk/d1/d2/d3", [], ["f3"]), ] assert result == expected def test_fileutils_walk_with_unicode_path(self): - test_dir = self.extract_test_zip('fileutils/walk/unicode.zip') - test_dir = join(test_dir, 'unicode') + test_dir = self.extract_test_zip("fileutils/walk/unicode.zip") + test_dir = join(test_dir, "unicode") result = list(x[-1] for x in fileutils.walk(test_dir)) - expected = [[u'2.csv'], [u'gru\u0308n.png']] + expected = [["2.csv"], ["gru\u0308n.png"]] assert result == expected def test_fileutils_walk_can_walk_a_single_file(self): - test_file = self.get_test_loc('fileutils/walk/f') + test_file = self.get_test_loc("fileutils/walk/f") result = list(fileutils.walk(test_file)) - expected = [ - (fileutils.parent_directory(test_file), [], ['f']) - ] + expected = [(fileutils.parent_directory(test_file), [], ["f"])] assert result == expected def test_fileutils_walk_can_walk_an_empty_dir(self): test_dir = self.get_temp_dir() result = list(fileutils.walk(test_dir)) - expected = [ - (test_dir, [], []) - ] + expected = [(test_dir, [], [])] assert result == expected - @skipIf(on_macos_14_or_higher, 'Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635') + @skipIf( + on_macos_14_or_higher, + "Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635", + ) def test_walk_can_walk_non_utf8_path_from_unicode_path(self): - test_dir = self.extract_test_tar_raw( - 'fileutils/walk_non_utf8/non_unicode.tgz') - test_dir = join(test_dir, 'non_unicode') + test_dir = self.extract_test_tar_raw("fileutils/walk_non_utf8/non_unicode.tgz") + test_dir = join(test_dir, "non_unicode") if not on_linux: test_dir = str(test_dir) @@ -347,60 +353,58 @@ def test_walk_can_walk_non_utf8_path_from_unicode_path(self): _dirpath, _dirnames, filenames = result assert len(filenames) == 18 - @skipIf(on_macos_14_or_higher, 'Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635') + @skipIf( + on_macos_14_or_higher, + "Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635", + ) def test_os_walk_can_walk_non_utf8_path_from_unicode_path(self): - test_dir = self.extract_test_tar_raw( - 'fileutils/walk_non_utf8/non_unicode.tgz') - test_dir = join(test_dir, 'non_unicode') + test_dir = self.extract_test_tar_raw("fileutils/walk_non_utf8/non_unicode.tgz") + test_dir = join(test_dir, "non_unicode") result = list(os.walk(test_dir))[0] _dirpath, _dirnames, filenames = result assert len(filenames) == 18 - @skipIf(on_windows, 'os.symlink does not work on Windows') + @skipIf(on_windows, "os.symlink does not work on Windows") def test_walk_on_symlinks(self): - test_dir = self.get_test_loc('symlink/walk', copy=True) + test_dir = self.get_test_loc("symlink/walk", copy=True) temp_dir = fileutils.get_temp_dir() - test_link = join(temp_dir, 'test-dir-link') + test_link = join(temp_dir, "test-dir-link") os.symlink(test_dir, test_link) results = list(fileutils.walk(test_link, follow_symlinks=True)) - results = [(os.path.basename(top), dirs, files) - for top, dirs, files in results] - expected = [ - ('test-dir-link', ['dir'], ['a']), - ('dir', [], ['b']) - ] + results = [(os.path.basename(top), dirs, files) for top, dirs, files in results] + expected = [("test-dir-link", ["dir"], ["a"]), ("dir", [], ["b"])] assert results == expected class TestFileUtilsIter(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_resource_iter(self): - test_dir = self.get_test_loc('fileutils/walk') - base = self.get_test_loc('fileutils') - result = [as_posixpath(f.replace(base, '')) - for f in fileutils.resource_iter(test_dir, with_dirs=False)] + test_dir = self.get_test_loc("fileutils/walk") + base = self.get_test_loc("fileutils") + result = [ + as_posixpath(f.replace(base, "")) + for f in fileutils.resource_iter(test_dir, with_dirs=False) + ] expected = [ - '/walk/f', - '/walk/unicode.zip', - '/walk/d1/f1', - '/walk/d1/d2/f2', - '/walk/d1/d2/d3/f3' + "/walk/f", + "/walk/unicode.zip", + "/walk/d1/f1", + "/walk/d1/d2/f2", + "/walk/d1/d2/d3/f3", ] assert sorted(result) == sorted(expected) def test_resource_iter_can_iterate_a_single_file(self): - test_file = self.get_test_loc('fileutils/walk/f') - result = [as_posixpath(f) for f in fileutils.resource_iter( - test_file, with_dirs=False)] + test_file = self.get_test_loc("fileutils/walk/f") + result = [as_posixpath(f) for f in fileutils.resource_iter(test_file, with_dirs=False)] expected = [as_posixpath(test_file)] assert result == expected def test_resource_iter_can_iterate_a_single_file_with_dirs(self): - test_file = self.get_test_loc('fileutils/walk/f') - result = [as_posixpath(f) for f in fileutils.resource_iter( - test_file, with_dirs=True)] + test_file = self.get_test_loc("fileutils/walk/f") + result = [as_posixpath(f) for f in fileutils.resource_iter(test_file, with_dirs=True)] expected = [as_posixpath(test_file)] assert result == expected @@ -417,231 +421,235 @@ def test_resource_iter_can_walk_an_empty_dir_with_dirs(self): assert result == expected def test_resource_iter_without_dir(self): - test_dir = self.get_test_loc('fileutils/walk') - base = self.get_test_loc('fileutils') - result = sorted([as_posixpath(f.replace(base, '')) - for f in fileutils.resource_iter(test_dir, with_dirs=False)]) + test_dir = self.get_test_loc("fileutils/walk") + base = self.get_test_loc("fileutils") + result = sorted( + [ + as_posixpath(f.replace(base, "")) + for f in fileutils.resource_iter(test_dir, with_dirs=False) + ] + ) expected = [ - '/walk/f', - '/walk/unicode.zip', - '/walk/d1/f1', - '/walk/d1/d2/f2', - '/walk/d1/d2/d3/f3' + "/walk/f", + "/walk/unicode.zip", + "/walk/d1/f1", + "/walk/d1/d2/f2", + "/walk/d1/d2/d3/f3", ] assert sorted(result) == sorted(expected) def test_resource_iter_with_dirs(self): - test_dir = self.get_test_loc('fileutils/walk') - base = self.get_test_loc('fileutils') - result = sorted([as_posixpath(f.replace(base, '')) - for f in fileutils.resource_iter(test_dir, with_dirs=True)]) + test_dir = self.get_test_loc("fileutils/walk") + base = self.get_test_loc("fileutils") + result = sorted( + [ + as_posixpath(f.replace(base, "")) + for f in fileutils.resource_iter(test_dir, with_dirs=True) + ] + ) expected = [ - '/walk/d1', - '/walk/d1/d2', - '/walk/d1/d2/d3', - '/walk/d1/d2/d3/f3', - '/walk/d1/d2/f2', - '/walk/d1/f1', - '/walk/f', - '/walk/unicode.zip' + "/walk/d1", + "/walk/d1/d2", + "/walk/d1/d2/d3", + "/walk/d1/d2/d3/f3", + "/walk/d1/d2/f2", + "/walk/d1/f1", + "/walk/f", + "/walk/unicode.zip", ] assert sorted(result) == sorted(expected) def test_resource_iter_return_byte_on_byte_input(self): - test_dir = self.get_test_loc('fileutils/walk') - base = self.get_test_loc('fileutils') - result = sorted([as_posixpath(f.replace(base, '')) - for f in fileutils.resource_iter(test_dir, with_dirs=True)]) + test_dir = self.get_test_loc("fileutils/walk") + base = self.get_test_loc("fileutils") + result = sorted( + [ + as_posixpath(f.replace(base, "")) + for f in fileutils.resource_iter(test_dir, with_dirs=True) + ] + ) expected = [ - '/walk/d1', - '/walk/d1/d2', - '/walk/d1/d2/d3', - '/walk/d1/d2/d3/f3', - '/walk/d1/d2/f2', - '/walk/d1/f1', - '/walk/f', - '/walk/unicode.zip' + "/walk/d1", + "/walk/d1/d2", + "/walk/d1/d2/d3", + "/walk/d1/d2/d3/f3", + "/walk/d1/d2/f2", + "/walk/d1/f1", + "/walk/f", + "/walk/unicode.zip", ] assert sorted(result) == sorted(expected) assert all(isinstance(p, str) for p in result) def test_resource_iter_return_unicode_on_unicode_input(self): - test_dir = self.get_test_loc('fileutils/walk') - base = str(self.get_test_loc('fileutils')) - result = sorted([as_posixpath(f.replace(base, '')) - for f in fileutils.resource_iter(test_dir, with_dirs=True)]) + test_dir = self.get_test_loc("fileutils/walk") + base = str(self.get_test_loc("fileutils")) + result = sorted( + [ + as_posixpath(f.replace(base, "")) + for f in fileutils.resource_iter(test_dir, with_dirs=True) + ] + ) expected = [ - u'/walk/d1', - u'/walk/d1/d2', - u'/walk/d1/d2/d3', - u'/walk/d1/d2/d3/f3', - u'/walk/d1/d2/f2', - u'/walk/d1/f1', - u'/walk/f', - u'/walk/unicode.zip' + "/walk/d1", + "/walk/d1/d2", + "/walk/d1/d2/d3", + "/walk/d1/d2/d3/f3", + "/walk/d1/d2/f2", + "/walk/d1/f1", + "/walk/f", + "/walk/unicode.zip", ] assert sorted(result) == sorted(expected) assert all(isinstance(p, str) for p in result) def test_resource_iter_can_walk_unicode_path_with_zip(self): - test_dir = self.extract_test_zip('fileutils/walk/unicode.zip') - test_dir = join(test_dir, 'unicode') + test_dir = self.extract_test_zip("fileutils/walk/unicode.zip") + test_dir = join(test_dir, "unicode") test_dir = str(test_dir) - result = sorted([p.replace(test_dir, '') - for p in fileutils.resource_iter(test_dir)]) + result = sorted([p.replace(test_dir, "") for p in fileutils.resource_iter(test_dir)]) if on_linux: - expected = [ - u'/2.csv', - u'/a', - u'/a/gru\u0308n.png' - ] + expected = ["/2.csv", "/a", "/a/gru\u0308n.png"] elif on_mac: - expected = [ - u'/2.csv', - u'/a', - u'/a/gru\u0308n.png' - ] + expected = ["/2.csv", "/a", "/a/gru\u0308n.png"] elif on_windows: - expected = [ - u'\\2.csv', - u'\\a', - u'\\a\\gru\u0308n.png' - ] + expected = ["\\2.csv", "\\a", "\\a\\gru\u0308n.png"] assert result == expected - @skipIf(on_macos_14_or_higher, 'Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635') + @skipIf( + on_macos_14_or_higher, + "Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635", + ) def test_resource_iter_can_walk_non_utf8_path_from_unicode_path_with_dirs(self): - test_dir = self.extract_test_tar_raw( - 'fileutils/walk_non_utf8/non_unicode.tgz') - test_dir = join(test_dir, 'non_unicode') + test_dir = self.extract_test_tar_raw("fileutils/walk_non_utf8/non_unicode.tgz") + test_dir = join(test_dir, "non_unicode") result = list(fileutils.resource_iter(test_dir, with_dirs=True)) assert len(result) == 18 - @skipIf(on_macos_14_or_higher, 'Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635') + @skipIf( + on_macos_14_or_higher, + "Cannot handle yet byte paths on macOS 10.14+. See https://github.com/nexB/scancode-toolkit/issues/1635", + ) def test_resource_iter_can_walk_non_utf8_path_from_unicode_path(self): - test_dir = self.extract_test_tar_raw( - 'fileutils/walk_non_utf8/non_unicode.tgz') - test_dir = join(test_dir, 'non_unicode') + test_dir = self.extract_test_tar_raw("fileutils/walk_non_utf8/non_unicode.tgz") + test_dir = join(test_dir, "non_unicode") result = list(fileutils.resource_iter(test_dir, with_dirs=False)) assert len(result) == 18 - @skipIf(on_windows, 'Symlinks do not work well on Windows') + @skipIf(on_windows, "Symlinks do not work well on Windows") def test_resource_iter_follow_symlinks(self): - test_dir = self.get_test_loc('symlink/walk', copy=True) + test_dir = self.get_test_loc("symlink/walk", copy=True) temp_dir = fileutils.get_temp_dir() - test_link = join(temp_dir, 'test-dir-link') + test_link = join(temp_dir, "test-dir-link") os.symlink(test_dir, test_link) - result = [os.path.basename(f) for f in fileutils.resource_iter( - test_dir, follow_symlinks=True)] - expected = [ - 'dir', - 'a', - 'b' + result = [ + os.path.basename(f) for f in fileutils.resource_iter(test_dir, follow_symlinks=True) ] + expected = ["dir", "a", "b"] assert sorted(result) == sorted(expected) class TestBaseName(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_file_base_name_on_path_and_location_1(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/.a/file' - expected_name = 'file' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/.a/file" + expected_name = "file" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_file_path_for_dot_file(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/.a/' - expected_name = '.a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/.a/" + expected_name = ".a" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_file_path_for_dot_file_with_extension(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/.a.b' - expected_name = '.a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/.a.b" + expected_name = ".a" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_file_path_for_file_with_unknown_composed_extension(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/a.tag.gz' - expected_name = 'a.tag' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/a.tag.gz" + expected_name = "a.tag" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_file_path_for_file_with_known_composed_extension(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/a.tar.gz' - expected_name = 'a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/a.tar.gz" + expected_name = "a" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_dir_path(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/' - expected_name = 'b' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/" + expected_name = "b" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_plain_file(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/f.a' - expected_name = 'f' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/f.a" + expected_name = "f" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_plain_file_with_parent_dir_extension(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'f.a/a.c' - expected_name = 'a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "f.a/a.c" + expected_name = "a" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_path_for_plain_dir(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/' - expected_name = 'a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/" + expected_name = "a" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_path_for_plain_dir_with_extension(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'f.a/' - expected_name = 'f.a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "f.a/" + expected_name = "f.a" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) assert result == expected_name def test_file_base_name_on_path_for_plain_file(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'tst' - expected_name = 'tst' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "tst" + expected_name = "tst" result = fileutils.file_base_name(test_file) assert result == expected_name result = fileutils.file_base_name(join(test_dir, test_file)) @@ -649,93 +657,93 @@ def test_file_base_name_on_path_for_plain_file(self): class TestFileName(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_file_name_on_path_and_location_1(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/.a/file' - expected_name = 'file' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/.a/file" + expected_name = "file" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_2(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/.a/' - expected_name = '.a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/.a/" + expected_name = ".a" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_3(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/.a.b' - expected_name = '.a.b' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/.a.b" + expected_name = ".a.b" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_4(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/a.tag.gz' - expected_name = 'a.tag.gz' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/a.tag.gz" + expected_name = "a.tag.gz" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_5(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/' - expected_name = 'b' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/" + expected_name = "b" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_6(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/f.a' - expected_name = 'f.a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/f.a" + expected_name = "f.a" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_7(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/' - expected_name = 'a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/" + expected_name = "a" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_8(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'f.a/a.c' - expected_name = 'a.c' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "f.a/a.c" + expected_name = "a.c" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_9(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'f.a/' - expected_name = 'f.a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "f.a/" + expected_name = "f.a" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_name_on_path_and_location_10(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'tst' - expected_name = 'tst' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "tst" + expected_name = "tst" result = fileutils.file_name(test_file) assert result == expected_name result = fileutils.file_name((os.path.join(test_dir, test_file))) @@ -743,268 +751,259 @@ def test_file_name_on_path_and_location_10(self): class TestFileExtension(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_file_extension_on_path_and_location_1(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/.a/file' - expected_name = '' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/.a/file" + expected_name = "" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_2(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/.a/' - expected_name = '' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/.a/" + expected_name = "" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_3(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/.a.b' - expected_name = '.b' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/.a.b" + expected_name = ".b" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_4(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/a.tag.gz' - expected_name = '.gz' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/a.tag.gz" + expected_name = ".gz" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_5(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/' - expected_name = '' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/" + expected_name = "" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_6(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/f.a' - expected_name = '.a' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/f.a" + expected_name = ".a" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_7(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/' - expected_name = '' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/" + expected_name = "" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_8(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'f.a/a.c' - expected_name = '.c' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "f.a/a.c" + expected_name = ".c" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_9(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'f.a/' - expected_name = '' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "f.a/" + expected_name = "" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_file_extension_on_path_and_location_10(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'tst' - expected_name = '' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "tst" + expected_name = "" result = fileutils.file_extension(test_file) assert result == expected_name result = fileutils.file_extension((os.path.join(test_dir, test_file))) assert result == expected_name def test_splitext_base(self): - expected = 'path', '.ext' - assert fileutils.splitext('C:\\dir\\path.ext') == expected + expected = "path", ".ext" + assert fileutils.splitext("C:\\dir\\path.ext") == expected def test_splitext_directories_even_with_dotted_names_have_no_extension(self): import ntpath - expected = 'path.ext', '' - assert fileutils.splitext('C:\\dir\\path.ext' + ntpath.sep) == expected - expected = 'path.ext', '' - assert fileutils.splitext('/dir/path.ext/') == expected + expected = "path.ext", "" + assert fileutils.splitext("C:\\dir\\path.ext" + ntpath.sep) == expected + + expected = "path.ext", "" + assert fileutils.splitext("/dir/path.ext/") == expected - expected = 'file', '.txt' - assert fileutils.splitext('/some/file.txt') == expected + expected = "file", ".txt" + assert fileutils.splitext("/some/file.txt") == expected def test_splitext_composite_extensions_for_tarballs_are_properly_handled(self): - expected = 'archive', '.tar.gz' - assert fileutils.splitext('archive.tar.gz') == expected + expected = "archive", ".tar.gz" + assert fileutils.splitext("archive.tar.gz") == expected def test_splitext_name_base(self): - expected = 'path', '.ext' - assert fileutils.splitext_name('path.ext') == expected + expected = "path", ".ext" + assert fileutils.splitext_name("path.ext") == expected def test_splitext_name_directories_have_no_extension(self): - expected = 'path.ext', '' - assert fileutils.splitext_name('path.ext', is_file=False) == expected + expected = "path.ext", "" + assert fileutils.splitext_name("path.ext", is_file=False) == expected - expected = 'file', '.txt' - assert fileutils.splitext_name('file.txt') == expected + expected = "file", ".txt" + assert fileutils.splitext_name("file.txt") == expected def test_splitext_name_composite_extensions_for_tarballs_are_properly_handled(self): - expected = 'archive', '.tar.gz' - assert fileutils.splitext_name('archive.tar.gz') == expected + expected = "archive", ".tar.gz" + assert fileutils.splitext_name("archive.tar.gz") == expected def test_splitext_name_dotfile_are_properly_handled(self): - expected = '.dotfile', '' - assert fileutils.splitext_name('.dotfile') == expected - expected = '.dotfile', '.this' - assert fileutils.splitext_name('.dotfile.this') == expected + expected = ".dotfile", "" + assert fileutils.splitext_name(".dotfile") == expected + expected = ".dotfile", ".this" + assert fileutils.splitext_name(".dotfile.this") == expected class TestParentDir(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_parent_directory_on_path_and_location_1(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/.a/file' - expected_name = 'a/.a/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/.a/file" + expected_name = "a/.a/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_2(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/.a/' - expected_name = 'a/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/.a/" + expected_name = "a/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_3(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/.a.b' - expected_name = 'a/b/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/.a.b" + expected_name = "a/b/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_4(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/a.tag.gz' - expected_name = 'a/b/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/a.tag.gz" + expected_name = "a/b/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_5(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/b/' - expected_name = 'a/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/b/" + expected_name = "a/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_6(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/f.a' - expected_name = 'a/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/f.a" + expected_name = "a/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_7(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'a/' - expected_name = '/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "a/" + expected_name = "/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_8(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'f.a/a.c' - expected_name = 'f.a/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "f.a/a.c" + expected_name = "f.a/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_9(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'f.a/' - expected_name = '/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "f.a/" + expected_name = "/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) def test_parent_directory_on_path_and_location_10(self): - test_dir = self.get_test_loc('fileutils/basename') - test_file = 'tst' - expected_name = '/' + test_dir = self.get_test_loc("fileutils/basename") + test_file = "tst" + expected_name = "/" result = fileutils.parent_directory(test_file) result = fileutils.as_posixpath(result) assert result == expected_name - result = fileutils.parent_directory( - (os.path.join(test_dir, test_file))) + result = fileutils.parent_directory((os.path.join(test_dir, test_file))) result = fileutils.as_posixpath(result) assert result.endswith(expected_name) diff --git a/tests/test_functional.py b/tests/test_functional.py index 0f4b76c9..5953c03f 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -15,14 +15,12 @@ class TestFunctional(TestCase): - def test_flatten(self): - expected = [7, 6, 5, 4, 'a', 3, 3, 2, 1] + expected = [7, 6, 5, 4, "a", 3, 3, 2, 1] test = flatten([7, (6, [5, [4, ["a"], 3]], 3), 2, 1]) assert test == expected def test_flatten_generator(self): - def gen(): for _ in range(2): yield range(5) @@ -32,17 +30,16 @@ def gen(): assert test == expected def test_flatten_empties(self): - expected = ['a'] - test = flatten([[], (), ['a']]) + expected = ["a"] + test = flatten([[], (), ["a"]]) assert test == expected def test_partial(self): - def test_func(a, b): pass wrapped = partial(test_func, a=2) - assert wrapped.__name__ == 'test_func' + assert wrapped.__name__ == "test_func" def test_memoized(self): call_count = Counter() diff --git a/tests/test_hash.py b/tests/test_hash.py index 0ab15b52..27d28653 100644 --- a/tests/test_hash.py +++ b/tests/test_hash.py @@ -8,137 +8,157 @@ import os -from commoncode.testcase import FileBasedTesting - from commoncode.hash import b64sha1 from commoncode.hash import checksum from commoncode.hash import get_hasher from commoncode.hash import md5 from commoncode.hash import multi_checksums from commoncode.hash import sha1 +from commoncode.hash import sha1_git from commoncode.hash import sha256 from commoncode.hash import sha512 -from commoncode.hash import sha1_git +from commoncode.testcase import FileBasedTesting class TestHash(FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_get_hasher(self): h = get_hasher(160) - assert h(b'a').b64digest() == u'hvfkN_qlp_zhXR3cuerq6jd2Z7g=' - assert h(b'aa').b64digest() == u'4MkDWJjdUvxlxBRUzsnE0mEb-zc=' - assert h(b'aaa').b64digest() == u'fiQN50-x7Qj6CNOAY_amqRRiqBU=' + assert h(b"a").b64digest() == "hvfkN_qlp_zhXR3cuerq6jd2Z7g=" + assert h(b"aa").b64digest() == "4MkDWJjdUvxlxBRUzsnE0mEb-zc=" + assert h(b"aaa").b64digest() == "fiQN50-x7Qj6CNOAY_amqRRiqBU=" def test_short_hashes(self): h = get_hasher(32) - assert h(b'a').hexdigest() == u'0cc175b9' - assert h(b'aa').hexdigest() == u'4124bc0a' + assert h(b"a").hexdigest() == "0cc175b9" + assert h(b"aa").hexdigest() == "4124bc0a" h = get_hasher(64) - assert h(b'aa').hexdigest() == u'4124bc0a9335c27f' + assert h(b"aa").hexdigest() == "4124bc0a9335c27f" def test_sha1_checksum_on_text(self): - test_file = self.get_test_loc('hash/dir1/a.txt') - assert u'3ca69e8d6c234a469d16ac28a4a658c92267c423' == sha1(test_file) + test_file = self.get_test_loc("hash/dir1/a.txt") + assert "3ca69e8d6c234a469d16ac28a4a658c92267c423" == sha1(test_file) def test_sha1_checksum_on_text2(self): - test_file = self.get_test_loc('hash/dir2/a.txt') - assert u'3ca69e8d6c234a469d16ac28a4a658c92267c423' == sha1(test_file) + test_file = self.get_test_loc("hash/dir2/a.txt") + assert "3ca69e8d6c234a469d16ac28a4a658c92267c423" == sha1(test_file) def test_sha1_checksum_on_dos_text(self): - test_file = self.get_test_loc('hash/dir2/dos.txt') - assert u'a71718fb198630ae8ba32926015d8555a03cb06c' == sha1(test_file) + test_file = self.get_test_loc("hash/dir2/dos.txt") + assert "a71718fb198630ae8ba32926015d8555a03cb06c" == sha1(test_file) def test_sha1_checksum_base64(self): - test_file = self.get_test_loc('hash/dir1/a.png') - assert u'NKxUZdSKmwT8J18JvCIwZg349Pc=' == b64sha1(test_file) + test_file = self.get_test_loc("hash/dir1/a.png") + assert "NKxUZdSKmwT8J18JvCIwZg349Pc=" == b64sha1(test_file) def test_md5_checksum_on_text(self): - test_file = self.get_test_loc('hash/dir1/a.txt') - assert u'40c53c58fdafacc83cfff6ee3d2f6d69' == md5(test_file) + test_file = self.get_test_loc("hash/dir1/a.txt") + assert "40c53c58fdafacc83cfff6ee3d2f6d69" == md5(test_file) def test_md5_checksum_on_text2(self): - test_file = self.get_test_loc('hash/dir2/a.txt') - assert u'40c53c58fdafacc83cfff6ee3d2f6d69' == md5(test_file) + test_file = self.get_test_loc("hash/dir2/a.txt") + assert "40c53c58fdafacc83cfff6ee3d2f6d69" == md5(test_file) def test_md5_checksum_on_dos_text(self): - test_file = self.get_test_loc('hash/dir2/dos.txt') - assert u'095f5068940e41df9add5d4cc396c181' == md5(test_file) + test_file = self.get_test_loc("hash/dir2/dos.txt") + assert "095f5068940e41df9add5d4cc396c181" == md5(test_file) def test_md5_checksum(self): - test_file = self.get_test_loc('hash/dir1/a.png') - assert u'4760fb467f1ebf3b0aeace4a3926f1a4' == md5(test_file) + test_file = self.get_test_loc("hash/dir1/a.png") + assert "4760fb467f1ebf3b0aeace4a3926f1a4" == md5(test_file) def test_sha1_checksum(self): - test_file = self.get_test_loc('hash/dir1/a.png') - assert u'34ac5465d48a9b04fc275f09bc2230660df8f4f7' == sha1(test_file) + test_file = self.get_test_loc("hash/dir1/a.png") + assert "34ac5465d48a9b04fc275f09bc2230660df8f4f7" == sha1(test_file) def test_sha256_checksum(self): - test_file = self.get_test_loc('hash/dir1/a.png') - assert u'1b598db6fee8f1ec7bb919c0adf68956f3d20af8c9934a9cf2db52e1347efd35' == sha256( - test_file) + test_file = self.get_test_loc("hash/dir1/a.png") + assert "1b598db6fee8f1ec7bb919c0adf68956f3d20af8c9934a9cf2db52e1347efd35" == sha256( + test_file + ) def test_sha512_checksum(self): - test_file = self.get_test_loc('hash/dir1/a.png') - assert u'5be9e01cd20ff288fd3c3fc46be5c2747eaa2c526197125330947a95cdb418222176b182a4680f0e435ba8f114363c45a67b30eed9a9222407e63ccbde46d3b4' == sha512( - test_file) + test_file = self.get_test_loc("hash/dir1/a.png") + assert ( + "5be9e01cd20ff288fd3c3fc46be5c2747eaa2c526197125330947a95cdb418222176b182a4680f0e435ba8f114363c45a67b30eed9a9222407e63ccbde46d3b4" + == sha512(test_file) + ) def test_checksum_sha1(self): - test_file = self.get_test_loc('hash/dir1/a.txt') - assert '3ca69e8d6c234a469d16ac28a4a658c92267c423' == checksum( - test_file, 'sha1') + test_file = self.get_test_loc("hash/dir1/a.txt") + assert "3ca69e8d6c234a469d16ac28a4a658c92267c423" == checksum(test_file, "sha1") def test_checksum_md5(self): - test_file = self.get_test_loc('hash/dir1/a.txt') - assert '40c53c58fdafacc83cfff6ee3d2f6d69' == checksum(test_file, 'md5') + test_file = self.get_test_loc("hash/dir1/a.txt") + assert "40c53c58fdafacc83cfff6ee3d2f6d69" == checksum(test_file, "md5") def test_multi_checksums(self): - test_file = self.get_test_loc('hash/dir1/a.png') - expected = dict([ - ('md5', u'4760fb467f1ebf3b0aeace4a3926f1a4'), - ('sha1', u'34ac5465d48a9b04fc275f09bc2230660df8f4f7'), - ('sha256', u'1b598db6fee8f1ec7bb919c0adf68956f3d20af8c9934a9cf2db52e1347efd35'), - ]) - result = multi_checksums(test_file, 'md5 sha1 sha256'.split()) + test_file = self.get_test_loc("hash/dir1/a.png") + expected = dict( + [ + ("md5", "4760fb467f1ebf3b0aeace4a3926f1a4"), + ("sha1", "34ac5465d48a9b04fc275f09bc2230660df8f4f7"), + ("sha256", "1b598db6fee8f1ec7bb919c0adf68956f3d20af8c9934a9cf2db52e1347efd35"), + ] + ) + result = multi_checksums(test_file, "md5 sha1 sha256".split()) assert result == expected def test_multi_checksums_custom(self): - test_file = self.get_test_loc('hash/dir1/a.png') - result = multi_checksums(test_file, ('sha512',)) - expected = dict([ - ('sha512', u'5be9e01cd20ff288fd3c3fc46be5c2747eaa2c526197125330947a95cdb418222176b182a4680f0e435ba8f114363c45a67b30eed9a9222407e63ccbde46d3b4'), - ]) + test_file = self.get_test_loc("hash/dir1/a.png") + result = multi_checksums(test_file, ("sha512",)) + expected = dict( + [ + ( + "sha512", + "5be9e01cd20ff288fd3c3fc46be5c2747eaa2c526197125330947a95cdb418222176b182a4680f0e435ba8f114363c45a67b30eed9a9222407e63ccbde46d3b4", + ), + ] + ) assert result == expected def test_multi_checksums_shattered1(self): - test_file = self.get_test_loc('hash/sha1-collision/shattered-1.pdf') - expected = dict([ - ('md5', 'ee4aa52b139d925f8d8884402b0a750c'), - ('sha1', '38762cf7f55934b34d179ae6a4c80cadccbb7f0a'), - ('sha256', '2bb787a73e37352f92383abe7e2902936d1059ad9f1ba6daaa9c1e58ee6970d0'), - ('sha512', '3c19b2cbcf72f7f5b252ea31677b8f2323d6119e49bcc0fb55931d00132385f1e749bb24cbd68c04ac826ae8421802825d3587fe185abf709669bb9693f6b416'), - ('sha1_git', 'ba9aaa145ccd24ef760cf31c74d8f7ca1a2e47b0'), - ]) + test_file = self.get_test_loc("hash/sha1-collision/shattered-1.pdf") + expected = dict( + [ + ("md5", "ee4aa52b139d925f8d8884402b0a750c"), + ("sha1", "38762cf7f55934b34d179ae6a4c80cadccbb7f0a"), + ("sha256", "2bb787a73e37352f92383abe7e2902936d1059ad9f1ba6daaa9c1e58ee6970d0"), + ( + "sha512", + "3c19b2cbcf72f7f5b252ea31677b8f2323d6119e49bcc0fb55931d00132385f1e749bb24cbd68c04ac826ae8421802825d3587fe185abf709669bb9693f6b416", + ), + ("sha1_git", "ba9aaa145ccd24ef760cf31c74d8f7ca1a2e47b0"), + ] + ) result = multi_checksums(test_file) assert result == expected def test_multi_checksums_shattered2(self): - test_file = self.get_test_loc('hash/sha1-collision/shattered-2.pdf') - expected = dict([ - ('md5', '5bd9d8cabc46041579a311230539b8d1'), - ('sha1', '38762cf7f55934b34d179ae6a4c80cadccbb7f0a'), - ('sha256', 'd4488775d29bdef7993367d541064dbdda50d383f89f0aa13a6ff2e0894ba5ff'), - ('sha512', 'f39a04842e4b28e04558496beb7cb84654ded9c00b2f873c3ef64f9dfdbc760cd0273b816858ba5b203c0dd71af8b65d6a0c1032e00e48ace0b4705eedcc1bab'), - # Note: this is not the same as the sha1_git for shattered-1.pdf ;) - ('sha1_git', 'b621eeccd5c7edac9b7dcba35a8d5afd075e24f2'), - ]) + test_file = self.get_test_loc("hash/sha1-collision/shattered-2.pdf") + expected = dict( + [ + ("md5", "5bd9d8cabc46041579a311230539b8d1"), + ("sha1", "38762cf7f55934b34d179ae6a4c80cadccbb7f0a"), + ("sha256", "d4488775d29bdef7993367d541064dbdda50d383f89f0aa13a6ff2e0894ba5ff"), + ( + "sha512", + "f39a04842e4b28e04558496beb7cb84654ded9c00b2f873c3ef64f9dfdbc760cd0273b816858ba5b203c0dd71af8b65d6a0c1032e00e48ace0b4705eedcc1bab", + ), + # Note: this is not the same as the sha1_git for shattered-1.pdf ;) + ("sha1_git", "b621eeccd5c7edac9b7dcba35a8d5afd075e24f2"), + ] + ) result = multi_checksums(test_file) assert result == expected def test_sha1_git_checksum(self): # $ pushd tests/commoncode/data && for f in `find hash/ -type f` ; # do echo -n "$f ";git hash-object --literally $f; done && popd - tests = [t.strip().split() for t in ''' + tests = [ + t.strip().split() + for t in """ hash/dir1/a.txt de980441c3ab03a8c07dda1ad27b8a11f39deb1e hash/dir1/a.png 5f212358671a3ada8794cb14fb5227f596447a8c hash/sha1-collision/shattered-1.pdf ba9aaa145ccd24ef760cf31c74d8f7ca1a2e47b0 @@ -147,7 +167,9 @@ def test_sha1_git_checksum(self): hash/sha1-collision/shattered-2.pdf b621eeccd5c7edac9b7dcba35a8d5afd075e24f2 hash/dir2/dos.txt 0d2d3a69833f1ebcbf420875cfbc93f132bc8a0b hash/dir2/a.txt de980441c3ab03a8c07dda1ad27b8a11f39deb1e - '''.splitlines() if t.strip()] + """.splitlines() + if t.strip() + ] for test_file, expected_sha1_git in tests: test_file = self.get_test_loc(test_file) # test that we match the git hash-object diff --git a/tests/test_ignore.py b/tests/test_ignore.py index 984c512c..8c74286f 100644 --- a/tests/test_ignore.py +++ b/tests/test_ignore.py @@ -17,94 +17,98 @@ class IgnoreTest(commoncode.testcase.FileBasedTesting): - test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + test_data_dir = os.path.join(os.path.dirname(__file__), "data") - @skipIf(on_mac, 'Return different result on Mac for reasons to investigate') + @skipIf(on_mac, "Return different result on Mac for reasons to investigate") def test_is_ignored_default_ignores_eclipse1(self): - test_dir = self.extract_test_tar('ignore/excludes/eclipse.tgz') - test_base = os.path.join(test_dir, 'eclipse') + test_dir = self.extract_test_tar("ignore/excludes/eclipse.tgz") + test_base = os.path.join(test_dir, "eclipse") - test = os.path.join(test_base, '.settings') + test = os.path.join(test_base, ".settings") assert ignore.is_ignored(test, ignore.default_ignores, {}) def test_is_ignored_default_ignores_eclipse2(self): - test_dir = self.extract_test_tar('ignore/excludes/eclipse.tgz') - test_base = os.path.join(test_dir, 'eclipse') + test_dir = self.extract_test_tar("ignore/excludes/eclipse.tgz") + test_base = os.path.join(test_dir, "eclipse") - test = os.path.join(test_base, '.settings/somefile') + test = os.path.join(test_base, ".settings/somefile") assert ignore.is_ignored(test, ignore.default_ignores, {}) def test_is_ignored_default_ignores_eclipse3(self): - test_dir = self.extract_test_tar('ignore/excludes/eclipse.tgz') - test_base = os.path.join(test_dir, 'eclipse') + test_dir = self.extract_test_tar("ignore/excludes/eclipse.tgz") + test_base = os.path.join(test_dir, "eclipse") - test = os.path.join(test_base, '.project') + test = os.path.join(test_base, ".project") assert ignore.is_ignored(test, ignore.default_ignores, {}) def test_is_ignored_default_ignores_eclipse4(self): - test_dir = self.extract_test_tar('ignore/excludes/eclipse.tgz') - test_base = os.path.join(test_dir, 'eclipse') + test_dir = self.extract_test_tar("ignore/excludes/eclipse.tgz") + test_base = os.path.join(test_dir, "eclipse") - test = os.path.join(test_base, '.pydevproject') + test = os.path.join(test_base, ".pydevproject") assert ignore.is_ignored(test, ignore.default_ignores, {}) def test_is_ignored_default_ignores_mac1(self): - test_dir = self.extract_test_tar('ignore/excludes/mac.tgz') - test_base = os.path.join(test_dir, 'mac') + test_dir = self.extract_test_tar("ignore/excludes/mac.tgz") + test_base = os.path.join(test_dir, "mac") - test = os.path.join(test_base, '__MACOSX') + test = os.path.join(test_base, "__MACOSX") assert ignore.is_ignored(test, ignore.default_ignores, {}) def test_is_ignored_default_ignores_mac2(self): - test_dir = self.extract_test_tar('ignore/excludes/mac.tgz') - test_base = os.path.join(test_dir, 'mac') + test_dir = self.extract_test_tar("ignore/excludes/mac.tgz") + test_base = os.path.join(test_dir, "mac") - test = os.path.join( - test_base, '__MACOSX/comp_match/smallrepo/._jetty_1.0_index.csv') + test = os.path.join(test_base, "__MACOSX/comp_match/smallrepo/._jetty_1.0_index.csv") assert ignore.is_ignored(test, ignore.default_ignores, {}) def test_is_ignored_default_ignores_mac3(self): - test_dir = self.extract_test_tar('ignore/excludes/mac.tgz') - test_base = os.path.join(test_dir, 'mac') + test_dir = self.extract_test_tar("ignore/excludes/mac.tgz") + test_base = os.path.join(test_dir, "mac") - test = os.path.join(test_base, '.DS_Store') + test = os.path.join(test_base, ".DS_Store") assert ignore.is_ignored(test, ignore.default_ignores, {}) def test_is_ignored_default_ignores_mac4(self): - test_dir = self.extract_test_tar('ignore/excludes/mac.tgz') - test_base = os.path.join(test_dir, 'mac') + test_dir = self.extract_test_tar("ignore/excludes/mac.tgz") + test_base = os.path.join(test_dir, "mac") - test = os.path.join(test_base, '.DS_Store/a') + test = os.path.join(test_base, ".DS_Store/a") assert ignore.is_ignored(test, ignore.default_ignores, {}) - @skipIf(on_mac, 'Return different result on Mac for reasons to investigate') + @skipIf(on_mac, "Return different result on Mac for reasons to investigate") def test_is_ignored_default_ignores_mac5(self): - test_dir = self.extract_test_tar('ignore/excludes/mac.tgz') - test_base = os.path.join(test_dir, 'mac') + test_dir = self.extract_test_tar("ignore/excludes/mac.tgz") + test_base = os.path.join(test_dir, "mac") - test = os.path.join(test_base, '._.DS_Store') + test = os.path.join(test_base, "._.DS_Store") # this is really weird as a behavior # 'Default ignore: MacOSX artifact' assert ignore.is_ignored(test, ignore.default_ignores, {}) - @skipIf(on_mac, 'Return different result on Mac for reasons to investigate') + @skipIf(on_mac, "Return different result on Mac for reasons to investigate") def test_is_ignored_default_ignores_msft(self): - test_dir = self.extract_test_tar('ignore/excludes/msft-vs.tgz') - test = os.path.join(test_dir, 'msft-vs/tst.sluo') + test_dir = self.extract_test_tar("ignore/excludes/msft-vs.tgz") + test = os.path.join(test_dir, "msft-vs/tst.sluo") # 'Default ignore: Microsoft VS project artifact' ?? assert ignore.is_ignored(test, ignore.default_ignores, {}) - @skipIf(on_mac, 'Return different result on Mac for reasons to investigate') + @skipIf(on_mac, "Return different result on Mac for reasons to investigate") def test_is_ignored_skip_vcs_files_and_dirs(self): - test_dir = self.extract_test_tar('ignore/vcs.tgz') + test_dir = self.extract_test_tar("ignore/vcs.tgz") result = [] for top, dirs, files in os.walk(test_dir, topdown=True): not_ignored = [] for d in dirs: p = os.path.join(top, d) ign = ignore.is_ignored(p, ignore.default_ignores, {}) - tp = fileutils.as_posixpath(p.replace(test_dir, '')) - result.append((tp, ign,)) + tp = fileutils.as_posixpath(p.replace(test_dir, "")) + result.append( + ( + tp, + ign, + ) + ) if not ign: not_ignored.append(d) @@ -114,94 +118,90 @@ def test_is_ignored_skip_vcs_files_and_dirs(self): for f in files: p = os.path.join(top, f) ign = ignore.is_ignored(p, ignore.default_ignores, {}) - tp = fileutils.as_posixpath(p.replace(test_dir, '')) - result.append((tp, ign,)) + tp = fileutils.as_posixpath(p.replace(test_dir, "")) + result.append( + ( + tp, + ign, + ) + ) expected = [ - ('/vcs', False), - ('/vcs/.bzr', True), - ('/vcs/.git', True), - ('/vcs/.hg', True), - ('/vcs/.repo', True), - ('/vcs/.svn', True), - ('/vcs/CVS', True), - ('/vcs/_darcs', True), - ('/vcs/_MTN', True), - ('/vcs/.bzrignore', True), - ('/vcs/.cvsignore', True), - ('/vcs/.gitignore', True), - ('/vcs/.hgignore', True), - ('/vcs/.svnignore', True), - ('/vcs/vssver.scc', True), + ("/vcs", False), + ("/vcs/.bzr", True), + ("/vcs/.git", True), + ("/vcs/.hg", True), + ("/vcs/.repo", True), + ("/vcs/.svn", True), + ("/vcs/CVS", True), + ("/vcs/_darcs", True), + ("/vcs/_MTN", True), + ("/vcs/.bzrignore", True), + ("/vcs/.cvsignore", True), + ("/vcs/.gitignore", True), + ("/vcs/.hgignore", True), + ("/vcs/.svnignore", True), + ("/vcs/vssver.scc", True), ] assert sorted(result) == sorted(expected) def test_fileset_is_included_with_default_ignore_does_not_skip_one_char_names(self): # use fileset directly to work on strings not locations from commoncode import fileset - tests = [c for c in 'HFS+ Private Data'] + 'HFS+ Private Data'.split() - result = [(t, - fileset.is_included(t, excludes=ignore.default_ignores, includes={})) - for t in tests] + + tests = [c for c in "HFS+ Private Data"] + "HFS+ Private Data".split() + result = [ + (t, fileset.is_included(t, excludes=ignore.default_ignores, includes={})) for t in tests + ] expected = [ - ('H', True), - ('F', True), - ('S', True), - ('+', True), - (' ', False), - ('P', True), - ('r', True), - ('i', True), - ('v', True), - ('a', True), - ('t', True), - ('e', True), - (' ', False), - ('D', True), - ('a', True), - ('t', True), - ('a', True), - ('HFS+', True), - ('Private', True), - ('Data', True) + ("H", True), + ("F", True), + ("S", True), + ("+", True), + (" ", False), + ("P", True), + ("r", True), + ("i", True), + ("v", True), + ("a", True), + ("t", True), + ("e", True), + (" ", False), + ("D", True), + ("a", True), + ("t", True), + ("a", True), + ("HFS+", True), + ("Private", True), + ("Data", True), ] assert result == expected - @skipIf(on_mac or on_windows, 'We are only testing on posix for now') + @skipIf(on_mac or on_windows, "We are only testing on posix for now") def test_is_ignored_path_string_skip_special(self): - test_path = '/test/path' - assert ignore.is_ignored( - test_path, {'asdf': 'skip'}, {}, skip_special=True) - assert not ignore.is_ignored( - test_path, {'asdf': 'skip'}, {}, skip_special=False) + test_path = "/test/path" + assert ignore.is_ignored(test_path, {"asdf": "skip"}, {}, skip_special=True) + assert not ignore.is_ignored(test_path, {"asdf": "skip"}, {}, skip_special=False) - @skipIf(on_mac or on_windows, 'We are only testing on posix for now') + @skipIf(on_mac or on_windows, "We are only testing on posix for now") def test_is_ignored_special_files_skip_special(self): test_fifo = self.get_temp_file() os.mkfifo(test_fifo) - assert ignore.is_ignored( - test_fifo, {'asdf': 'skip'}, {}, skip_special=True) - assert not ignore.is_ignored( - test_fifo, {'asdf': 'skip'}, {}, skip_special=False) + assert ignore.is_ignored(test_fifo, {"asdf": "skip"}, {}, skip_special=True) + assert not ignore.is_ignored(test_fifo, {"asdf": "skip"}, {}, skip_special=False) test_symlink = self.get_temp_file() - test_file_location = self.get_test_loc('ignore/vcs.tgz') + test_file_location = self.get_test_loc("ignore/vcs.tgz") os.symlink(test_file_location, test_symlink) - assert ignore.is_ignored( - test_symlink, {'asdf': 'skip'}, {}, skip_special=True) - assert not ignore.is_ignored( - test_symlink, {'asdf': 'skip'}, {}, skip_special=False) + assert ignore.is_ignored(test_symlink, {"asdf": "skip"}, {}, skip_special=True) + assert not ignore.is_ignored(test_symlink, {"asdf": "skip"}, {}, skip_special=False) - @skipIf(on_mac or on_windows, 'We are only testing on posix for now') + @skipIf(on_mac or on_windows, "We are only testing on posix for now") def test_is_ignored_real_location_skip_special(self): - test_file_location = self.get_test_loc('ignore/vcs.tgz') - assert not ignore.is_ignored( - test_file_location, {'asdf': 'skip'}, {}, skip_special=True) - assert not ignore.is_ignored( - test_file_location, {'asdf': 'skip'}, {}, skip_special=False) - - assert not ignore.is_ignored( - test_file_location, {'asdf': 'skip'}, {}, skip_special=True) - assert not ignore.is_ignored( - test_file_location, {'asdf': 'skip'}, {}, skip_special=False) + test_file_location = self.get_test_loc("ignore/vcs.tgz") + assert not ignore.is_ignored(test_file_location, {"asdf": "skip"}, {}, skip_special=True) + assert not ignore.is_ignored(test_file_location, {"asdf": "skip"}, {}, skip_special=False) + + assert not ignore.is_ignored(test_file_location, {"asdf": "skip"}, {}, skip_special=True) + assert not ignore.is_ignored(test_file_location, {"asdf": "skip"}, {}, skip_special=False) diff --git a/tests/test_paths.py b/tests/test_paths.py index c9ca8ffa..67bff071 100644 --- a/tests/test_paths.py +++ b/tests/test_paths.py @@ -12,319 +12,315 @@ class TestPortablePath(TestCase): - def test_safe_path_mixed_slashes(self): test = paths.safe_path( - 'C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\drupal6/drupal.js') - expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/drupal.js' + "C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\drupal6/drupal.js" + ) + expected = "C/Documents_and_Settings/Boki/Desktop/head/patches/drupal6/drupal.js" assert test == expected def test_safe_path_mixed_slashes_and_spaces(self): test = paths.safe_path( - 'C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\parallel uploads/drupal.js') - expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/parallel_uploads/drupal.js' + "C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\parallel uploads/drupal.js" + ) + expected = "C/Documents_and_Settings/Boki/Desktop/head/patches/parallel_uploads/drupal.js" assert test == expected def test_safe_path_windows_style(self): test = paths.safe_path( - 'C:\\Documents and Settings\\Administrator\\Desktop\\siftDemoV4_old\\defs.h') - expected = 'C/Documents_and_Settings/Administrator/Desktop/siftDemoV4_old/defs.h' + "C:\\Documents and Settings\\Administrator\\Desktop\\siftDemoV4_old\\defs.h" + ) + expected = "C/Documents_and_Settings/Administrator/Desktop/siftDemoV4_old/defs.h" assert test == expected def test_safe_path_windows_style_mixed_slashes_no_spaces(self): test = paths.safe_path( - 'C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\imagefield/imagefield.css') - expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/imagefield/imagefield.css' + "C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\imagefield/imagefield.css" + ) + expected = "C/Documents_and_Settings/Boki/Desktop/head/patches/imagefield/imagefield.css" assert test == expected def test_safe_path_windows_style_spaces(self): test = paths.safe_path( - 'C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\js delete\\imagefield.css') - expected = 'C/Documents_and_Settings/Boki/Desktop/head/patches/js_delete/imagefield.css' + "C:\\Documents and Settings\\Boki\\Desktop\\head\\patches\\js delete\\imagefield.css" + ) + expected = "C/Documents_and_Settings/Boki/Desktop/head/patches/js_delete/imagefield.css" assert test == expected def test_safe_path_windows_style_posix_slashes(self): test = paths.safe_path( - 'C:/Documents and Settings/Alex Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java') - expected = 'C/Documents_and_Settings/Alex_Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java' + "C:/Documents and Settings/Alex Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java" + ) + expected = "C/Documents_and_Settings/Alex_Burgel/workspace/Hibernate3.2/test/org/hibernate/test/AllTests.java" assert test == expected def test_safe_path_windows_style_relative(self): - test = paths.safe_path('includes\\webform.components.inc') - expected = 'includes/webform.components.inc' + test = paths.safe_path("includes\\webform.components.inc") + expected = "includes/webform.components.inc" assert test == expected def test_safe_path_windows_style_absolute_trailing_slash(self): - test = paths.safe_path('\\includes\\webform.components.inc\\') - expected = 'includes/webform.components.inc' + test = paths.safe_path("\\includes\\webform.components.inc\\") + expected = "includes/webform.components.inc" assert test == expected def test_safe_path_posix_style_relative(self): - test = paths.safe_path('includes/webform.components.inc') - expected = 'includes/webform.components.inc' + test = paths.safe_path("includes/webform.components.inc") + expected = "includes/webform.components.inc" assert test == expected def test_safe_path_posix_style_absolute_trailing_slash(self): - test = paths.safe_path('/includes/webform.components.inc/') - expected = 'includes/webform.components.inc' + test = paths.safe_path("/includes/webform.components.inc/") + expected = "includes/webform.components.inc" assert test == expected def test_safe_path_posix_style_french_char(self): - test = paths.safe_path('/includes/webform.compon\xc3nts.inc/') - expected = 'includes/webform.componAnts.inc' + test = paths.safe_path("/includes/webform.compon\xc3nts.inc/") + expected = "includes/webform.componAnts.inc" assert test == expected def test_safe_path_posix_style_chinese_char(self): - test = paths.safe_path(b'/includes/webform.compon\xd2\xaants.inc/') + test = paths.safe_path(b"/includes/webform.compon\xd2\xaants.inc/") expected = [ - 'includes/webform.componNSnts.inc', - 'includes/webform.componS_nts.inc', + "includes/webform.componNSnts.inc", + "includes/webform.componS_nts.inc", ] assert test in expected def test_safe_path_windows_style_dots(self): - test = paths.safe_path('\\includes\\..\\webform.components.inc\\') - expected = 'webform.components.inc' + test = paths.safe_path("\\includes\\..\\webform.components.inc\\") + expected = "webform.components.inc" assert test == expected def test_safe_path_windows_style_many_dots(self): - test = paths.safe_path( - '.\\includes\\.\\..\\..\\..\\webform.components.inc\\.') - expected = 'dotdot/dotdot/webform.components.inc' + test = paths.safe_path(".\\includes\\.\\..\\..\\..\\webform.components.inc\\.") + expected = "dotdot/dotdot/webform.components.inc" assert test == expected def test_safe_path_posix_style_dots(self): - test = paths.safe_path('includes/../webform.components.inc') - expected = 'webform.components.inc' + test = paths.safe_path("includes/../webform.components.inc") + expected = "webform.components.inc" assert test == expected def test_safe_path_posix_style_many_dots(self): - test = paths.safe_path( - './includes/./../../../../webform.components.inc/.') - expected = 'dotdot/dotdot/dotdot/webform.components.inc' + test = paths.safe_path("./includes/./../../../../webform.components.inc/.") + expected = "dotdot/dotdot/dotdot/webform.components.inc" assert test == expected def test_safe_path_posix_only(self): - test_path = 'var/lib/dpkg/info/libgsm1:amd64.list' + test_path = "var/lib/dpkg/info/libgsm1:amd64.list" test = paths.safe_path(test_path) - expected = 'var/lib/dpkg/info/libgsm1_amd64.list' + expected = "var/lib/dpkg/info/libgsm1_amd64.list" assert test == expected test = paths.safe_path(test_path, posix_only=True) assert test == test_path def test_resolve_mixed_slash(self): - test = paths.resolve('C:\\..\\./drupal.js') - expected = 'C/drupal.js' + test = paths.resolve("C:\\..\\./drupal.js") + expected = "C/drupal.js" assert test == expected def test_resolve_2(self): - test = paths.resolve('\\includes\\..\\webform.components.inc\\') - expected = 'webform.components.inc' + test = paths.resolve("\\includes\\..\\webform.components.inc\\") + expected = "webform.components.inc" assert test == expected def test_resolve_3(self): - test = paths.resolve('includes/../webform.components.inc') - expected = 'webform.components.inc' + test = paths.resolve("includes/../webform.components.inc") + expected = "webform.components.inc" assert test == expected def test_resolve_4(self): - test = paths.resolve( - '////.//includes/./../..//..///../webform.components.inc/.') - expected = 'dotdot/dotdot/dotdot/webform.components.inc' + test = paths.resolve("////.//includes/./../..//..///../webform.components.inc/.") + expected = "dotdot/dotdot/dotdot/webform.components.inc" assert test == expected def test_resolve_5(self): - test = paths.resolve( - u'////.//includes/./../..//..///../webform.components.inc/.') - expected = u'dotdot/dotdot/dotdot/webform.components.inc' + test = paths.resolve("////.//includes/./../..//..///../webform.components.inc/.") + expected = "dotdot/dotdot/dotdot/webform.components.inc" assert test == expected def test_resolve_6(self): - test = paths.resolve('includes/../') - expected = '.' + test = paths.resolve("includes/../") + expected = "." assert test == expected def test_portable_filename(self): - expected = 'A___file__with_Spaces.mov' - assert paths.portable_filename( - "A:\\ file/ with Spaces.mov") == expected + expected = "A___file__with_Spaces.mov" + assert paths.portable_filename("A:\\ file/ with Spaces.mov") == expected # Test `preserve_spaces` option. Spaces should not be replaced - expected = 'Program Files (x86)' - assert paths.portable_filename( - "Program Files (x86)", preserve_spaces=True) == expected + expected = "Program Files (x86)" + assert paths.portable_filename("Program Files (x86)", preserve_spaces=True) == expected # Unresolved relative paths will be treated as a single filename. Use # resolve instead if you want to resolve paths: - expected = '___.._.._etc_passwd' + expected = "___.._.._etc_passwd" assert paths.portable_filename("../../../etc/passwd") == expected # Unicode name are transliterated: - expected = 'This_contain_UMLAUT_umlauts.txt' - assert paths.portable_filename( - u'This contain UMLAUT \xfcml\xe4uts.txt') == expected + expected = "This_contain_UMLAUT_umlauts.txt" + assert paths.portable_filename("This contain UMLAUT \xfcml\xe4uts.txt") == expected # Check to see if illegal Windows filenames are properly handled for illegal_window_name in paths.ILLEGAL_WINDOWS_NAMES: # Rename files with names that are illegal on Windows - expected = f'{illegal_window_name}_' + expected = f"{illegal_window_name}_" assert paths.portable_filename(illegal_window_name) == expected # Allow files with names that are illegal on Windows - assert paths.portable_filename( - illegal_window_name, posix_only=True) == illegal_window_name + assert ( + paths.portable_filename(illegal_window_name, posix_only=True) == illegal_window_name + ) # Check to see if the posix_only option does and does not replace # punctuation characters that are illegal in Windows filenames for valid_posix_path_char in paths.posix_legal_punctuation: - test_name = f'test{valid_posix_path_char}' - assert paths.portable_filename( - test_name, posix_only=True) == test_name + test_name = f"test{valid_posix_path_char}" + assert paths.portable_filename(test_name, posix_only=True) == test_name if valid_posix_path_char not in paths.legal_punctuation: - expected = f'test_' + expected = f"test_" assert paths.portable_filename(test_name) == expected class TestCommonPath(TestCase): - def test_common_path_prefix1(self): - test = paths.common_path_prefix('/a/b/c', '/a/b/c') - assert test == ('a/b/c', 3) + test = paths.common_path_prefix("/a/b/c", "/a/b/c") + assert test == ("a/b/c", 3) def test_common_path_prefix2(self): - test = paths.common_path_prefix('/a/b/c', '/a/b') - assert test == ('a/b', 2) + test = paths.common_path_prefix("/a/b/c", "/a/b") + assert test == ("a/b", 2) def test_common_path_prefix3(self): - test = paths.common_path_prefix('/a/b', '/a/b/c') - assert test == ('a/b', 2) + test = paths.common_path_prefix("/a/b", "/a/b/c") + assert test == ("a/b", 2) def test_common_path_prefix4(self): - test = paths.common_path_prefix('/a', '/a') - assert test == ('a', 1) + test = paths.common_path_prefix("/a", "/a") + assert test == ("a", 1) def test_common_path_prefix_path_root(self): - test = paths.common_path_prefix('/a/b/c', '/') + test = paths.common_path_prefix("/a/b/c", "/") assert test == (None, 0) def test_common_path_prefix_root_path(self): - test = paths.common_path_prefix('/', '/a/b/c') + test = paths.common_path_prefix("/", "/a/b/c") assert test == (None, 0) def test_common_path_prefix_root_root(self): - test = paths.common_path_prefix('/', '/') + test = paths.common_path_prefix("/", "/") assert test == (None, 0) def test_common_path_prefix_path_elements_are_similar(self): - test = paths.common_path_prefix('/a/b/c', '/a/b/d') - assert test == ('a/b', 2) + test = paths.common_path_prefix("/a/b/c", "/a/b/d") + assert test == ("a/b", 2) def test_common_path_prefix_no_match(self): - test = paths.common_path_prefix('/abc/d', '/abe/f') + test = paths.common_path_prefix("/abc/d", "/abe/f") assert test == (None, 0) def test_common_path_prefix_ignore_training_slashes(self): - test = paths.common_path_prefix('/a/b/c/', '/a/b/c/') - assert test == ('a/b/c', 3) + test = paths.common_path_prefix("/a/b/c/", "/a/b/c/") + assert test == ("a/b/c", 3) def test_common_path_prefix8(self): - test = paths.common_path_prefix('/a/b/c/', '/a/b') - assert test == ('a/b', 2) + test = paths.common_path_prefix("/a/b/c/", "/a/b") + assert test == ("a/b", 2) def test_common_path_prefix10(self): - test = paths.common_path_prefix('/a/b/c.txt', '/a/b/b.txt') - assert test == ('a/b', 2) + test = paths.common_path_prefix("/a/b/c.txt", "/a/b/b.txt") + assert test == ("a/b", 2) def test_common_path_prefix11(self): - test = paths.common_path_prefix('/a/b/c.txt', '/a/b.txt') - assert test == ('a', 1) + test = paths.common_path_prefix("/a/b/c.txt", "/a/b.txt") + assert test == ("a", 1) def test_common_path_prefix12(self): - test = paths.common_path_prefix('/a/c/e/x.txt', '/a/d/a.txt') - assert test == ('a', 1) + test = paths.common_path_prefix("/a/c/e/x.txt", "/a/d/a.txt") + assert test == ("a", 1) def test_common_path_prefix13(self): - test = paths.common_path_prefix('/a/c/e/x.txt', '/a/d/') - assert test == ('a', 1) + test = paths.common_path_prefix("/a/c/e/x.txt", "/a/d/") + assert test == ("a", 1) def test_common_path_prefix14(self): - test = paths.common_path_prefix('/a/c/e/', '/a/d/') - assert test == ('a', 1) + test = paths.common_path_prefix("/a/c/e/", "/a/d/") + assert test == ("a", 1) def test_common_path_prefix15(self): - test = paths.common_path_prefix('/a/c/e/', '/a/c/a.txt') - assert test == ('a/c', 2) + test = paths.common_path_prefix("/a/c/e/", "/a/c/a.txt") + assert test == ("a/c", 2) def test_common_path_prefix16(self): - test = paths.common_path_prefix('/a/c/e/', '/a/c/f/') - assert test == ('a/c', 2) + test = paths.common_path_prefix("/a/c/e/", "/a/c/f/") + assert test == ("a/c", 2) def test_common_path_prefix17(self): - test = paths.common_path_prefix('/a/a.txt', '/a/b.txt/') - assert test == ('a', 1) + test = paths.common_path_prefix("/a/a.txt", "/a/b.txt/") + assert test == ("a", 1) def test_common_path_prefix18(self): - test = paths.common_path_prefix('/a/c/', '/a/') - assert test == ('a', 1) + test = paths.common_path_prefix("/a/c/", "/a/") + assert test == ("a", 1) def test_common_path_prefix19(self): - test = paths.common_path_prefix('/a/c.txt', '/a/') - assert test == ('a', 1) + test = paths.common_path_prefix("/a/c.txt", "/a/") + assert test == ("a", 1) def test_common_path_prefix20(self): - test = paths.common_path_prefix('/a/c/', '/a/d/') - assert test == ('a', 1) + test = paths.common_path_prefix("/a/c/", "/a/d/") + assert test == ("a", 1) def test_common_path_suffix(self): - test = paths.common_path_suffix('/a/b/c', '/a/b/c') - assert test == ('a/b/c', 3) + test = paths.common_path_suffix("/a/b/c", "/a/b/c") + assert test == ("a/b/c", 3) def test_common_path_suffix_absolute_relative(self): - test = paths.common_path_suffix('a/b/c', '/a/b/c') - assert test == ('a/b/c', 3) + test = paths.common_path_suffix("a/b/c", "/a/b/c") + assert test == ("a/b/c", 3) def test_common_path_suffix_find_subpath(self): - test = paths.common_path_suffix('/z/b/c', '/a/b/c') - assert test == ('b/c', 2) + test = paths.common_path_suffix("/z/b/c", "/a/b/c") + assert test == ("b/c", 2) def test_common_path_suffix_handles_relative_path(self): - test = paths.common_path_suffix('a/b', 'a/b') - assert test == ('a/b', 2) + test = paths.common_path_suffix("a/b", "a/b") + assert test == ("a/b", 2) def test_common_path_suffix_handles_relative_subpath(self): - test = paths.common_path_suffix('zsds/adsds/a/b/b/c', 'a//a/d//b/c') - assert test == ('b/c', 2) + test = paths.common_path_suffix("zsds/adsds/a/b/b/c", "a//a/d//b/c") + assert test == ("b/c", 2) def test_common_path_suffix_ignore_and_strip_trailing_slash(self): - test = paths.common_path_suffix('zsds/adsds/a/b/b/c/', 'a//a/d//b/c/') - assert test == ('b/c', 2) + test = paths.common_path_suffix("zsds/adsds/a/b/b/c/", "a//a/d//b/c/") + assert test == ("b/c", 2) def test_common_path_suffix_return_None_if_no_common_suffix(self): - test = paths.common_path_suffix('/a/b/c', '/') + test = paths.common_path_suffix("/a/b/c", "/") assert test == (None, 0) def test_common_path_suffix_return_None_if_no_common_suffix2(self): - test = paths.common_path_suffix('/', '/a/b/c') + test = paths.common_path_suffix("/", "/a/b/c") assert test == (None, 0) def test_common_path_suffix_match_only_whole_segments(self): # only segments are honored, commonality within segment is ignored - test = paths.common_path_suffix( - 'this/is/aaaa/great/path', 'this/is/aaaaa/great/path') - assert test == ('great/path', 2) + test = paths.common_path_suffix("this/is/aaaa/great/path", "this/is/aaaaa/great/path") + assert test == ("great/path", 2) def test_common_path_suffix_two_root(self): - test = paths.common_path_suffix('/', '/') + test = paths.common_path_suffix("/", "/") assert test == (None, 0) def test_common_path_suffix_empty_root(self): - test = paths.common_path_suffix('', '/') + test = paths.common_path_suffix("", "/") assert test == (None, 0) def test_common_path_suffix_root_empty(self): - test = paths.common_path_suffix('/', '') + test = paths.common_path_suffix("/", "") assert test == (None, 0) def test_common_path_suffix_empty_empty(self): - test = paths.common_path_suffix('', '') + test = paths.common_path_suffix("", "") assert test == (None, 0) diff --git a/tests/test_resource.py b/tests/test_resource.py index 25f2bd95..9711b660 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -24,21 +24,21 @@ class TestCodebase(FileBasedTesting): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_walk_defaults(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) results = list(codebase.walk()) expected = [ - ('codebase', False), - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("codebase", False), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected @@ -47,79 +47,79 @@ def test_Codebase_do_not_ignore_by_default_older_sccs_and_rcs_dirs(self): from commoncode.fileutils import create_dir test_codebase = self.get_temp_dir() - create_dir(join(test_codebase, 'sccs', 'a')) - create_dir(join(test_codebase, 'rcs', 'b')) + create_dir(join(test_codebase, "sccs", "a")) + create_dir(join(test_codebase, "rcs", "b")) codebase = Codebase(test_codebase) results = list(codebase.walk(topdown=True, skip_root=True)) - expected = ['rcs', 'b', 'sccs', 'a'] + expected = ["rcs", "b", "sccs", "a"] assert [r.name for r in results] == expected def test_walk_topdown(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) results = list(codebase.walk(topdown=True)) expected = [ - ('codebase', False), - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("codebase", False), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_walk_bottomup(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) results = list(codebase.walk(topdown=False)) expected = [ - ('abc', True), - ('et131x.h', True), - ('that', True), - ('this', True), - ('dir', False), - ('file', True), - ('other dir', False), - ('codebase', False), + ("abc", True), + ("et131x.h", True), + ("that", True), + ("this", True), + ("dir", False), + ("file", True), + ("other dir", False), + ("codebase", False), ] assert [(r.name, r.is_file) for r in results] == expected def test_walk_skip_root_basic(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) results = list(codebase.walk(skip_root=True)) expected = [ - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_walk_filtered_with_filtered_root(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) codebase.root.is_filtered = True codebase.save_resource(codebase.root) results = list(codebase.walk_filtered()) expected = [ - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_walk_filtered_with_all_filtered(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) for res in codebase.walk(): res.is_filtered = True @@ -130,14 +130,14 @@ def test_walk_filtered_with_all_filtered(self): assert [(r.name, r.is_file) for r in results] == expected def test_compute_counts_filtered_None(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) results = codebase.compute_counts(skip_filtered=True) expected = (5, 3, 0) assert results == expected def test_compute_counts_filtered_None_with_size(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) for res in codebase.walk(): if res.is_file: @@ -149,14 +149,14 @@ def test_compute_counts_filtered_None_with_size(self): assert results == expected def test_compute_counts_filtered_None_with_cache(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) results = codebase.compute_counts(skip_filtered=True) expected = (5, 3, 0) assert results == expected def test_compute_counts_filtered_all(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) for res in codebase.walk(): res.is_filtered = True @@ -166,7 +166,7 @@ def test_compute_counts_filtered_all(self): assert results == expected def test_compute_counts_filtered_all_with_cache(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) for res in codebase.walk(): res.is_filtered = True @@ -176,7 +176,7 @@ def test_compute_counts_filtered_all_with_cache(self): assert results == expected def test_compute_counts_filtered_files(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) for res in codebase.walk(): if res.is_file: @@ -187,7 +187,7 @@ def test_compute_counts_filtered_files(self): assert results == expected def test_compute_counts_filtered_dirs(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) for res in codebase.walk(): if not res.is_file: @@ -198,7 +198,7 @@ def test_compute_counts_filtered_dirs(self): assert results == expected def test_walk_filtered_dirs(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) for res in codebase.walk(): if not res.is_file: @@ -207,33 +207,33 @@ def test_walk_filtered_dirs(self): results = list(codebase.walk_filtered(topdown=True)) expected = [ - ('abc', True), - ('et131x.h', True), - ('that', True), - ('this', True), - ('file', True), + ("abc", True), + ("et131x.h", True), + ("that", True), + ("this", True), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_walk_filtered_skip_root(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) codebase.root.is_filtered = True codebase.save_resource(codebase.root) results = list(codebase.walk_filtered(skip_root=True)) expected = [ - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_walk_filtered_all_skip_root(self): - test_codebase = self.get_test_loc('resource/codebase') + test_codebase = self.get_test_loc("resource/codebase") codebase = Codebase(test_codebase) for res in codebase.walk(): res.is_filtered = True @@ -243,21 +243,21 @@ def test_walk_filtered_all_skip_root(self): assert [(r.name, r.is_file) for r in results] == expected def test_walk_skip_root_single_file(self): - test_codebase = self.get_test_loc('resource/codebase/et131x.h') + test_codebase = self.get_test_loc("resource/codebase/et131x.h") codebase = Codebase(test_codebase) results = list(codebase.walk(skip_root=True)) - expected = [('et131x.h', True)] + expected = [("et131x.h", True)] assert [(r.name, r.is_file) for r in results] == expected def test_walk_filtered_with_skip_root_and_single_file_not_filtered(self): - test_codebase = self.get_test_loc('resource/codebase/et131x.h') + test_codebase = self.get_test_loc("resource/codebase/et131x.h") codebase = Codebase(test_codebase) results = list(codebase.walk_filtered(skip_root=True)) - expected = [('et131x.h', True)] + expected = [("et131x.h", True)] assert [(r.name, r.is_file) for r in results] == expected def test_walk_filtered__with_skip_root_and_filtered_single_file(self): - test_codebase = self.get_test_loc('resource/codebase/et131x.h') + test_codebase = self.get_test_loc("resource/codebase/et131x.h") codebase = Codebase(test_codebase) codebase.root.is_filtered = True codebase.save_resource(codebase.root) @@ -266,55 +266,51 @@ def test_walk_filtered__with_skip_root_and_filtered_single_file(self): assert [(r.name, r.is_file) for r in results] == expected def test_walk_skip_root_single_file_with_children(self): - test_codebase = self.get_test_loc('resource/codebase/et131x.h') + test_codebase = self.get_test_loc("resource/codebase/et131x.h") codebase = Codebase(test_codebase, strip_root=True) root = codebase.root - c1 = codebase._get_or_create_resource( - 'child1', parent=root, is_file=True) - codebase._get_or_create_resource('child2', parent=c1, is_file=False) + c1 = codebase._get_or_create_resource("child1", parent=root, is_file=True) + codebase._get_or_create_resource("child2", parent=c1, is_file=False) results = list(codebase.walk(skip_root=True)) - expected = [('et131x.h', True), ('child1', True), ('child2', False)] + expected = [("et131x.h", True), ("child1", True), ("child2", False)] assert [(r.name, r.is_file) for r in results] == expected def test_walk_filtered_with_skip_root_and_single_file_with_children(self): - test_codebase = self.get_test_loc('resource/codebase/et131x.h') + test_codebase = self.get_test_loc("resource/codebase/et131x.h") codebase = Codebase(test_codebase, strip_root=True) - c1 = codebase._get_or_create_resource( - 'some child', parent=codebase.root, is_file=True) - c2 = codebase._get_or_create_resource( - 'some child2', parent=c1, is_file=False) + c1 = codebase._get_or_create_resource("some child", parent=codebase.root, is_file=True) + c2 = codebase._get_or_create_resource("some child2", parent=c1, is_file=False) c2.is_filtered = True codebase.save_resource(c2) results = list(codebase.walk_filtered(skip_root=True)) - expected = [('et131x.h', True), ('some child', True)] + expected = [("et131x.h", True), ("some child", True)] assert [(r.name, r.is_file) for r in results] == expected c1.is_filtered = True codebase.save_resource(c1) results = list(codebase.walk_filtered(skip_root=True)) - expected = [('et131x.h', True)] + expected = [("et131x.h", True)] assert [(r.name, r.is_file) for r in results] == expected def test_walk_skip_root_single_dir(self): - test_codebase = self.get_temp_dir('walk') + test_codebase = self.get_temp_dir("walk") codebase = Codebase(test_codebase, strip_root=True) results = list(codebase.walk(skip_root=True)) - expected = [('walk', False)] + expected = [("walk", False)] assert [(r.name, r.is_file) for r in results] == expected def test_walk_skipped_directories_should_not_be_yielded(self): # Resources that we continue past should not be added to the result list - test_codebase = self.get_test_loc( - 'resource/skip_directories_during_walk') + test_codebase = self.get_test_loc("resource/skip_directories_during_walk") cdbs = Codebase(test_codebase) def _ignored(resource, codebase): - return resource.is_dir and resource.name == 'skip-this-directory' + return resource.is_dir and resource.name == "skip-this-directory" result = [ res.name @@ -324,47 +320,44 @@ def _ignored(resource, codebase): ) ] - expected = ['skip_directories_during_walk', 'this-should-be-returned'] + expected = ["skip_directories_during_walk", "this-should-be-returned"] assert result == expected def test__create_resource_can_add_child_to_file(self): - test_codebase = self.get_test_loc('resource/codebase/et131x.h') + test_codebase = self.get_test_loc("resource/codebase/et131x.h") codebase = Codebase(test_codebase) - codebase._get_or_create_resource( - 'some child', codebase.root, is_file=True) + codebase._get_or_create_resource("some child", codebase.root, is_file=True) results = list(codebase.walk()) - expected = [('et131x.h', True), ('some child', True)] + expected = [("et131x.h", True), ("some child", True)] assert [(r.name, r.is_file) for r in results] == expected def test__create_resource_can_add_child_to_dir(self): - test_codebase = self.get_temp_dir('resource') + test_codebase = self.get_temp_dir("resource") codebase = Codebase(test_codebase) - codebase._get_or_create_resource( - 'some child', codebase.root, is_file=False) + codebase._get_or_create_resource("some child", codebase.root, is_file=False) results = list(codebase.walk()) - expected = [('resource', False), ('some child', False)] + expected = [("resource", False), ("some child", False)] assert [(r.name, r.is_file) for r in results] == expected def test_get_resource_for_single_resource_codebase(self): - test_codebase = self.get_temp_dir('resource') + test_codebase = self.get_temp_dir("resource") codebase = Codebase(test_codebase) - assert not (codebase.root is codebase.get_resource('resource')) - assert codebase.get_resource('resource') == codebase.root + assert not (codebase.root is codebase.get_resource("resource")) + assert codebase.get_resource("resource") == codebase.root def test_get_resource_for_multiple_resource_codebase(self): - test_codebase = self.get_temp_dir('resource') - for name in ('a', 'b', 'c'): - with open(os.path.join(test_codebase, name), 'w') as o: - o.write('\n') + test_codebase = self.get_temp_dir("resource") + for name in ("a", "b", "c"): + with open(os.path.join(test_codebase, name), "w") as o: + o.write("\n") codebase = Codebase(test_codebase) - assert codebase.get_resource('resource/a').path == 'resource/a' - assert codebase.get_resource('/resource/c').path == 'resource/c' - assert codebase.get_resource( - 'resource/dsasda/../b/').path == 'resource/b' + assert codebase.get_resource("resource/a").path == "resource/a" + assert codebase.get_resource("/resource/c").path == "resource/c" + assert codebase.get_resource("resource/dsasda/../b/").path == "resource/b" def test_Resource_build_path(self): - test_dir = self.get_test_loc('resource/samples') + test_dir = self.get_test_loc("resource/samples") locations = [] for top, dirs, files in os.walk(test_dir): for x in dirs: @@ -376,49 +369,49 @@ def test_Resource_build_path(self): resources_no_root = list(codebase.walk(skip_root=True)) expected_default = [ - 'samples/JGroups', - 'samples/zlib', - 'samples/arch', - 'samples/README', - 'samples/screenshot.png', - 'samples/JGroups/src', - 'samples/JGroups/licenses', - 'samples/JGroups/LICENSE', - 'samples/JGroups/EULA', - 'samples/JGroups/src/GuardedBy.java', - 'samples/JGroups/src/ImmutableReference.java', - 'samples/JGroups/src/RouterStub.java', - 'samples/JGroups/src/S3_PING.java', - 'samples/JGroups/src/FixedMembershipToken.java', - 'samples/JGroups/src/RouterStubManager.java', - 'samples/JGroups/src/RATE_LIMITER.java', - 'samples/JGroups/licenses/cpl-1.0.txt', - 'samples/JGroups/licenses/bouncycastle.txt', - 'samples/JGroups/licenses/lgpl.txt', - 'samples/JGroups/licenses/apache-2.0.txt', - 'samples/JGroups/licenses/apache-1.1.txt', - 'samples/zlib/dotzlib', - 'samples/zlib/iostream2', - 'samples/zlib/infback9', - 'samples/zlib/gcc_gvmat64', - 'samples/zlib/ada', - 'samples/zlib/deflate.h', - 'samples/zlib/zutil.c', - 'samples/zlib/zlib.h', - 'samples/zlib/deflate.c', - 'samples/zlib/zutil.h', - 'samples/zlib/adler32.c', - 'samples/zlib/dotzlib/AssemblyInfo.cs', - 'samples/zlib/dotzlib/LICENSE_1_0.txt', - 'samples/zlib/dotzlib/readme.txt', - 'samples/zlib/dotzlib/ChecksumImpl.cs', - 'samples/zlib/iostream2/zstream_test.cpp', - 'samples/zlib/iostream2/zstream.h', - 'samples/zlib/infback9/infback9.c', - 'samples/zlib/infback9/infback9.h', - 'samples/zlib/gcc_gvmat64/gvmat64.S', - 'samples/zlib/ada/zlib.ads', - 'samples/arch/zlib.tar.gz', + "samples/JGroups", + "samples/zlib", + "samples/arch", + "samples/README", + "samples/screenshot.png", + "samples/JGroups/src", + "samples/JGroups/licenses", + "samples/JGroups/LICENSE", + "samples/JGroups/EULA", + "samples/JGroups/src/GuardedBy.java", + "samples/JGroups/src/ImmutableReference.java", + "samples/JGroups/src/RouterStub.java", + "samples/JGroups/src/S3_PING.java", + "samples/JGroups/src/FixedMembershipToken.java", + "samples/JGroups/src/RouterStubManager.java", + "samples/JGroups/src/RATE_LIMITER.java", + "samples/JGroups/licenses/cpl-1.0.txt", + "samples/JGroups/licenses/bouncycastle.txt", + "samples/JGroups/licenses/lgpl.txt", + "samples/JGroups/licenses/apache-2.0.txt", + "samples/JGroups/licenses/apache-1.1.txt", + "samples/zlib/dotzlib", + "samples/zlib/iostream2", + "samples/zlib/infback9", + "samples/zlib/gcc_gvmat64", + "samples/zlib/ada", + "samples/zlib/deflate.h", + "samples/zlib/zutil.c", + "samples/zlib/zlib.h", + "samples/zlib/deflate.c", + "samples/zlib/zutil.h", + "samples/zlib/adler32.c", + "samples/zlib/dotzlib/AssemblyInfo.cs", + "samples/zlib/dotzlib/LICENSE_1_0.txt", + "samples/zlib/dotzlib/readme.txt", + "samples/zlib/dotzlib/ChecksumImpl.cs", + "samples/zlib/iostream2/zstream_test.cpp", + "samples/zlib/iostream2/zstream.h", + "samples/zlib/infback9/infback9.c", + "samples/zlib/infback9/infback9.h", + "samples/zlib/gcc_gvmat64/gvmat64.S", + "samples/zlib/ada/zlib.ads", + "samples/arch/zlib.tar.gz", ] default = sorted( @@ -427,49 +420,49 @@ def test_Resource_build_path(self): assert default == sorted(expected_default) expected_strip_root = [ - 'JGroups', - 'zlib', - 'arch', - 'README', - 'screenshot.png', - 'JGroups/src', - 'JGroups/licenses', - 'JGroups/LICENSE', - 'JGroups/EULA', - 'JGroups/src/GuardedBy.java', - 'JGroups/src/ImmutableReference.java', - 'JGroups/src/RouterStub.java', - 'JGroups/src/S3_PING.java', - 'JGroups/src/FixedMembershipToken.java', - 'JGroups/src/RouterStubManager.java', - 'JGroups/src/RATE_LIMITER.java', - 'JGroups/licenses/cpl-1.0.txt', - 'JGroups/licenses/bouncycastle.txt', - 'JGroups/licenses/lgpl.txt', - 'JGroups/licenses/apache-2.0.txt', - 'JGroups/licenses/apache-1.1.txt', - 'zlib/dotzlib', - 'zlib/iostream2', - 'zlib/infback9', - 'zlib/gcc_gvmat64', - 'zlib/ada', - 'zlib/deflate.h', - 'zlib/zutil.c', - 'zlib/zlib.h', - 'zlib/deflate.c', - 'zlib/zutil.h', - 'zlib/adler32.c', - 'zlib/dotzlib/AssemblyInfo.cs', - 'zlib/dotzlib/LICENSE_1_0.txt', - 'zlib/dotzlib/readme.txt', - 'zlib/dotzlib/ChecksumImpl.cs', - 'zlib/iostream2/zstream_test.cpp', - 'zlib/iostream2/zstream.h', - 'zlib/infback9/infback9.c', - 'zlib/infback9/infback9.h', - 'zlib/gcc_gvmat64/gvmat64.S', - 'zlib/ada/zlib.ads', - 'arch/zlib.tar.gz', + "JGroups", + "zlib", + "arch", + "README", + "screenshot.png", + "JGroups/src", + "JGroups/licenses", + "JGroups/LICENSE", + "JGroups/EULA", + "JGroups/src/GuardedBy.java", + "JGroups/src/ImmutableReference.java", + "JGroups/src/RouterStub.java", + "JGroups/src/S3_PING.java", + "JGroups/src/FixedMembershipToken.java", + "JGroups/src/RouterStubManager.java", + "JGroups/src/RATE_LIMITER.java", + "JGroups/licenses/cpl-1.0.txt", + "JGroups/licenses/bouncycastle.txt", + "JGroups/licenses/lgpl.txt", + "JGroups/licenses/apache-2.0.txt", + "JGroups/licenses/apache-1.1.txt", + "zlib/dotzlib", + "zlib/iostream2", + "zlib/infback9", + "zlib/gcc_gvmat64", + "zlib/ada", + "zlib/deflate.h", + "zlib/zutil.c", + "zlib/zlib.h", + "zlib/deflate.c", + "zlib/zutil.h", + "zlib/adler32.c", + "zlib/dotzlib/AssemblyInfo.cs", + "zlib/dotzlib/LICENSE_1_0.txt", + "zlib/dotzlib/readme.txt", + "zlib/dotzlib/ChecksumImpl.cs", + "zlib/iostream2/zstream_test.cpp", + "zlib/iostream2/zstream.h", + "zlib/infback9/infback9.c", + "zlib/infback9/infback9.h", + "zlib/gcc_gvmat64/gvmat64.S", + "zlib/ada/zlib.ads", + "arch/zlib.tar.gz", ] stripped = sorted(r.strip_root_path for r in resources_no_root) assert stripped == sorted(expected_strip_root) @@ -480,7 +473,7 @@ def test_Resource_build_path(self): assert full_loc.endswith((ending)) def test_compute_counts_when_using_disk_cache(self): - test_codebase = self.get_test_loc('resource/samples') + test_codebase = self.get_test_loc("resource/samples") codebase = Codebase(test_codebase, strip_root=True, max_in_memory=-1) files_count, dirs_count, size_count = codebase.compute_counts() assert 33 == files_count @@ -488,35 +481,34 @@ def test_compute_counts_when_using_disk_cache(self): assert 0 == size_count def test_distance(self): - test_dir = self.get_test_loc('resource/dist') + test_dir = self.get_test_loc("resource/dist") codebase = Codebase(test_dir) assert codebase.root.distance(test_dir) == 0 - res = codebase.get_resource('dist/JGroups') - assert res.name == 'JGroups' + res = codebase.get_resource("dist/JGroups") + assert res.name == "JGroups" assert res.distance(codebase) == 1 - res = codebase.get_resource('dist/simple/META-INF/MANIFEST.MF') - assert res.name == 'MANIFEST.MF' - assert res.full_root_path.endswith( - 'resource/dist/simple/META-INF/MANIFEST.MF') + res = codebase.get_resource("dist/simple/META-INF/MANIFEST.MF") + assert res.name == "MANIFEST.MF" + assert res.full_root_path.endswith("resource/dist/simple/META-INF/MANIFEST.MF") assert res.distance(codebase) == 3 def test_skip_files_and_subdirs_of_ignored_dirs(self): - test_dir = self.get_test_loc('resource/ignore') + test_dir = self.get_test_loc("resource/ignore") codebase = Codebase(test_dir) # The `cvs` directory should not be visited - expected = ['ignore', 'ignore/file1'] + expected = ["ignore", "ignore/file1"] result = [r.path for r in codebase.walk(topdown=True)] self.assertEqual(expected, result) def test_depth_negative_fails(self): - test_codebase = self.get_test_loc('resource/deeply_nested') + test_codebase = self.get_test_loc("resource/deeply_nested") with self.assertRaises(Exception): next(depth_walk(test_codebase, -1)) def test_depth_walk_with_depth_0(self): - test_codebase = self.get_test_loc('resource/deeply_nested') + test_codebase = self.get_test_loc("resource/deeply_nested") results_zero = list(depth_walk(test_codebase, 0)) results_neg = list(depth_walk(test_codebase, float("inf"))) result_zero_dirs = [i for j in results_zero for i in j[1]] @@ -527,170 +519,165 @@ def test_depth_walk_with_depth_0(self): self.assertEqual(result_neg_files, result_zero_files) def test_depth_walk_with_depth_1(self): - test_codebase = self.get_test_loc('resource/deeply_nested') + test_codebase = self.get_test_loc("resource/deeply_nested") results = list(depth_walk(test_codebase, 1)) result_dirs = [i for j in results for i in j[1]].sort() result_files = [i for j in results for i in j[2]].sort() - expected_files = ['level1_file1', 'level1_file2'].sort() - expected_dirs = ['level1_dir1', 'level1_dir2'].sort() + expected_files = ["level1_file1", "level1_file2"].sort() + expected_dirs = ["level1_dir1", "level1_dir2"].sort() self.assertEqual(result_dirs, expected_dirs) self.assertEqual(result_files, expected_files) def test_depth_walk_with_depth_2(self): - test_codebase = self.get_test_loc('resource/deeply_nested') + test_codebase = self.get_test_loc("resource/deeply_nested") results = list(depth_walk(test_codebase, 2)) result_dirs = [i for j in results for i in j[1]].sort() result_files = [i for j in results for i in j[2]].sort() expected_files = [ - 'level1_file1', - 'level1_file2', - 'level2_file2', - 'level2_file1', - 'level2_file3', - 'level2_file4', - 'level2_file5', + "level1_file1", + "level1_file2", + "level2_file2", + "level2_file1", + "level2_file3", + "level2_file4", + "level2_file5", ].sort() - expected_dirs = ['level1_dir1', 'level1_dir2', - 'level2_dir1', 'level2_dir3'].sort() + expected_dirs = ["level1_dir1", "level1_dir2", "level2_dir1", "level2_dir3"].sort() self.assertEqual(result_dirs, expected_dirs) self.assertEqual(result_files, expected_files) def test_depth_walk_with_depth_3(self): - test_codebase = self.get_test_loc('resource/deeply_nested') + test_codebase = self.get_test_loc("resource/deeply_nested") results = list(depth_walk(test_codebase, 3)) result_dirs = [i for j in results for i in j[1]].sort() result_files = [i for j in results for i in j[2]].sort() expected_files = [ - 'level1_file1', - 'level1_file2', - 'level2_file2', - 'level2_file1', - 'level3_file2', - 'level3_file1', - 'level2_file3', - 'level2_file4', - 'level2_file5', - 'level3_file4', - 'level3_file3', + "level1_file1", + "level1_file2", + "level2_file2", + "level2_file1", + "level3_file2", + "level3_file1", + "level2_file3", + "level2_file4", + "level2_file5", + "level3_file4", + "level3_file3", ].sort() expected_dirs = [ - 'level1_dir1', - 'level1_dir2', - 'level2_dir1', - 'level3_dir1', - 'level2_dir3', + "level1_dir1", + "level1_dir2", + "level2_dir1", + "level3_dir1", + "level2_dir3", ].sort() self.assertEqual(result_dirs, expected_dirs) self.assertEqual(result_files, expected_files) def test_specify_depth_1(self): - test_codebase = self.get_test_loc('resource/deeply_nested') + test_codebase = self.get_test_loc("resource/deeply_nested") codebase = Codebase(test_codebase, max_depth=1) results = list(codebase.walk()) expected = [ - ('deeply_nested', False), - ('level1_dir1', False), - ('level1_dir2', False), - ('level1_file1', True), - ('level1_file2', True), + ("deeply_nested", False), + ("level1_dir1", False), + ("level1_dir2", False), + ("level1_file1", True), + ("level1_file2", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_specify_depth_2(self): - test_codebase = self.get_test_loc('resource/deeply_nested') + test_codebase = self.get_test_loc("resource/deeply_nested") codebase = Codebase(test_codebase, max_depth=2) results = list(codebase.walk()) expected = [ - ('deeply_nested', False), - ('level1_file1', True), - ('level1_file2', True), - ('level1_dir1', False), - ('level2_dir1', False), - ('level2_file1', True), - ('level2_file2', True), - ('level1_dir2', False), - ('level2_dir3', False), - ('level2_file3', True), - ('level2_file4', True), - ('level2_file5', True), + ("deeply_nested", False), + ("level1_file1", True), + ("level1_file2", True), + ("level1_dir1", False), + ("level2_dir1", False), + ("level2_file1", True), + ("level2_file2", True), + ("level1_dir2", False), + ("level2_dir3", False), + ("level2_file3", True), + ("level2_file4", True), + ("level2_file5", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_specify_depth_3(self): - test_codebase = self.get_test_loc('resource/deeply_nested') + test_codebase = self.get_test_loc("resource/deeply_nested") codebase = Codebase(test_codebase, max_depth=3) results = list(codebase.walk()) expected = [ - ('deeply_nested', False), - ('level1_file1', True), - ('level1_file2', True), - ('level1_dir1', False), - ('level2_file1', True), - ('level2_file2', True), - ('level2_dir1', False), - ('level3_dir1', False), - ('level3_file1', True), - ('level3_file2', True), - ('level1_dir2', False), - ('level2_file3', True), - ('level2_file4', True), - ('level2_file5', True), - ('level2_dir3', False), - ('level3_file3', True), - ('level3_file4', True), + ("deeply_nested", False), + ("level1_file1", True), + ("level1_file2", True), + ("level1_dir1", False), + ("level2_file1", True), + ("level2_file2", True), + ("level2_dir1", False), + ("level3_dir1", False), + ("level3_file1", True), + ("level3_file2", True), + ("level1_dir2", False), + ("level2_file3", True), + ("level2_file4", True), + ("level2_file5", True), + ("level2_dir3", False), + ("level3_file3", True), + ("level3_file4", True), ] assert [(r.name, r.is_file) for r in results] == expected class TestCodebaseWithPath(FileBasedTesting): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_Codebase_with_paths_works(self): - test_codebase = self.get_test_loc('resource/with_path/codebase') - paths = ['codebase/other dir/file'] + test_codebase = self.get_test_loc("resource/with_path/codebase") + paths = ["codebase/other dir/file"] codebase = Codebase(location=test_codebase, paths=paths) assert not codebase.errors results = [r.to_dict() for r in codebase.walk()] print(r.path for r in codebase.walk()) expected_file = self.get_test_loc( - 'resource/with_path/codebase-expected.json', + "resource/with_path/codebase-expected.json", must_exist=False, ) check_against_expected_json_file(results, expected_file, regen=False) def test_VirtualCodebase_with_paths_works(self): - test_codebase = self.get_test_loc( - 'resource/with_path/virtual-codebase.json') - paths = ['codebase/other dir/file'] + test_codebase = self.get_test_loc("resource/with_path/virtual-codebase.json") + paths = ["codebase/other dir/file"] codebase = VirtualCodebase(location=test_codebase, paths=paths) assert not codebase.errors results = [r.to_dict() for r in codebase.walk()] expected_file = self.get_test_loc( - 'resource/with_path/virtual-codebase-expected.json', + "resource/with_path/virtual-codebase-expected.json", must_exist=False, ) check_against_expected_json_file(results, expected_file, regen=False) def test_VirtualCodebase_codebase_attributes_assignment(self): - test_codebase = self.get_test_loc( - 'resource/with_path/virtual-codebase.json') + test_codebase = self.get_test_loc("resource/with_path/virtual-codebase.json") vc = VirtualCodebase( location=test_codebase, - codebase_attributes=dict( - packages=attr.ib(default=attr.Factory(list)) - ), + codebase_attributes=dict(packages=attr.ib(default=attr.Factory(list))), ) self.assertNotEqual(vc.attributes.packages, None) self.assertEqual(vc.attributes.packages, []) class TestCodebaseCache(FileBasedTesting): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_codebase_cache_default(self): - test_codebase = self.get_test_loc('resource/cache2') + test_codebase = self.get_test_loc("resource/cache2") codebase = Codebase(test_codebase) assert codebase.temp_dir @@ -702,15 +689,14 @@ def test_codebase_cache_default(self): assert not exists(cp) assert exists(parent_directory(cp)) - child = codebase._get_or_create_resource( - name='child', parent=root, is_file=True) + child = codebase._get_or_create_resource(name="child", parent=root, is_file=True) child.size = 12 codebase.save_resource(child) child_2 = codebase.get_resource(path=child.path) assert child_2 == child def test_codebase_cache_all_in_memory(self): - test_codebase = self.get_test_loc('resource/cache2') + test_codebase = self.get_test_loc("resource/cache2") codebase = Codebase(test_codebase, max_in_memory=0) for path, res in codebase.resources_by_path.items(): if res is Codebase.CACHED_RESOURCE: @@ -730,7 +716,7 @@ def test_codebase_cache_all_in_memory(self): ) def test_codebase_cache_all_on_disk(self): - test_codebase = self.get_test_loc('resource/cache2') + test_codebase = self.get_test_loc("resource/cache2") codebase = Codebase(test_codebase, max_in_memory=-1) for path, res in codebase.resources_by_path.items(): if res is Codebase.CACHED_RESOURCE: @@ -750,7 +736,7 @@ def test_codebase_cache_all_on_disk(self): ) def test_codebase_cache_mixed_two_in_memory(self): - test_codebase = self.get_test_loc('resource/cache2') + test_codebase = self.get_test_loc("resource/cache2") codebase = Codebase(test_codebase, max_in_memory=2) counter = 0 for path, res in codebase.resources_by_path.items(): @@ -778,115 +764,105 @@ def test_codebase_cache_mixed_two_in_memory(self): class TestVirtualCodebase(FileBasedTesting): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_virtual_codebase_walk_defaults(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + test_file = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") codebase = VirtualCodebase(location=test_file) results = list(codebase.walk()) expected = [ - ('codebase', False), - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("codebase", False), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_topdown(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + test_file = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") codebase = VirtualCodebase(location=test_file) results = list(codebase.walk(topdown=True)) expected = [ - ('codebase', False), - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("codebase", False), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_bottomup(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + test_file = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") codebase = VirtualCodebase(location=test_file) results = list(codebase.walk(topdown=False)) expected = [ - ('abc', True), - ('et131x.h', True), - ('that', True), - ('this', True), - ('dir', False), - ('file', True), - ('other dir', False), - ('codebase', False), + ("abc", True), + ("et131x.h", True), + ("that", True), + ("this", True), + ("dir", False), + ("file", True), + ("other dir", False), + ("codebase", False), ] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_skip_root_basic(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) results = list(virtual_codebase.walk(skip_root=True)) expected = [ - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_get_path_with_strip_root_and_walk_with_skip_root(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/stripped-and-skipped-root.json') + scan_data = self.get_test_loc("resource/virtual_codebase/stripped-and-skipped-root.json") virtual_codebase = VirtualCodebase(location=scan_data) - results = [r.get_path(strip_root=True) - for r in virtual_codebase.walk(skip_root=True)] - expected = ['README', 'screenshot.png'] + results = [r.get_path(strip_root=True) for r in virtual_codebase.walk(skip_root=True)] + expected = ["README", "screenshot.png"] assert expected == results def test_virtual_codebase_to_list_with_strip_root_and_walk_with_skip_root(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/stripped-and-skipped-root.json') + scan_data = self.get_test_loc("resource/virtual_codebase/stripped-and-skipped-root.json") virtual_codebase = VirtualCodebase(location=scan_data) results = virtual_codebase.to_list(strip_root=True, skinny=True) - expected = [{'path': 'README', 'type': 'file'}, - {'path': 'screenshot.png', 'type': 'file'}] + expected = [{"path": "README", "type": "file"}, {"path": "screenshot.png", "type": "file"}] assert expected == results def test_virtual_codebase_walk_filtered_with_filtered_root(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) virtual_codebase.root.is_filtered = True virtual_codebase.save_resource(virtual_codebase.root) results = list(virtual_codebase.walk_filtered()) expected = [ - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered_with_all_filtered(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): res.is_filtered = True @@ -897,16 +873,14 @@ def test_virtual_codebase_walk_filtered_with_all_filtered(self): assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_compute_counts_filtered_None(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) results = virtual_codebase.compute_counts(skip_filtered=True) expected = (5, 3, 2228) assert results == expected def test_virtual_codebase_compute_counts_filtered_None_with_size(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): if res.is_file: @@ -918,16 +892,14 @@ def test_virtual_codebase_compute_counts_filtered_None_with_size(self): assert results == expected def test_virtual_codebase_compute_counts_filtered_None_with_cache(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) results = virtual_codebase.compute_counts(skip_filtered=True) expected = (5, 3, 2228) assert results == expected def test_virtual_codebase_compute_counts_filtered_all(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): res.is_filtered = True @@ -937,8 +909,7 @@ def test_virtual_codebase_compute_counts_filtered_all(self): assert results == expected def test_virtual_codebase_compute_counts_filtered_all_with_cache(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): res.is_filtered = True @@ -948,8 +919,7 @@ def test_virtual_codebase_compute_counts_filtered_all_with_cache(self): assert results == expected def test_virtual_codebase_compute_counts_filtered_files(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): if res.is_file: @@ -960,8 +930,7 @@ def test_virtual_codebase_compute_counts_filtered_files(self): assert results == expected def test_virtual_codebase_compute_counts_filtered_dirs(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): if not res.is_file: @@ -972,8 +941,7 @@ def test_virtual_codebase_compute_counts_filtered_dirs(self): assert results == expected def test_virtual_codebase_walk_filtered_dirs(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): if not res.is_file: @@ -981,35 +949,33 @@ def test_virtual_codebase_walk_filtered_dirs(self): virtual_codebase.save_resource(res) results = list(virtual_codebase.walk_filtered(topdown=True)) expected = [ - ('abc', True), - ('et131x.h', True), - ('that', True), - ('this', True), - ('file', True), + ("abc", True), + ("et131x.h", True), + ("that", True), + ("this", True), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered_skip_root(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) virtual_codebase.root.is_filtered = True virtual_codebase.save_resource(virtual_codebase.root) results = list(virtual_codebase.walk_filtered(skip_root=True)) expected = [ - ('abc', True), - ('et131x.h', True), - ('dir', False), - ('that', True), - ('this', True), - ('other dir', False), - ('file', True), + ("abc", True), + ("et131x.h", True), + ("dir", False), + ("that", True), + ("this", True), + ("other dir", False), + ("file", True), ] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered_all_skip_root(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/virtual_codebase.json') + scan_data = self.get_test_loc("resource/virtual_codebase/virtual_codebase.json") virtual_codebase = VirtualCodebase(location=scan_data) for res in virtual_codebase.walk(): res.is_filtered = True @@ -1020,24 +986,21 @@ def test_virtual_codebase_walk_filtered_all_skip_root(self): assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_skip_root_single_file(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc("resource/virtual_codebase/et131x.h.json") virtual_codebase = VirtualCodebase(location=scan_data) results = list(virtual_codebase.walk(skip_root=True)) - expected = [('et131x.h', True)] + expected = [("et131x.h", True)] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered_with_skip_root_and_single_file_not_filtered(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc("resource/virtual_codebase/et131x.h.json") virtual_codebase = VirtualCodebase(location=scan_data) results = list(virtual_codebase.walk_filtered(skip_root=True)) - expected = [('et131x.h', True)] + expected = [("et131x.h", True)] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered__with_skip_root_and_filtered_single_file(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc("resource/virtual_codebase/et131x.h.json") virtual_codebase = VirtualCodebase(location=scan_data) virtual_codebase.root.is_filtered = True virtual_codebase.save_resource(virtual_codebase.root) @@ -1046,38 +1009,35 @@ def test_virtual_codebase_walk_filtered__with_skip_root_and_filtered_single_file assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_skip_root_single_file_with_children(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc("resource/virtual_codebase/et131x.h.json") virtual_codebase = VirtualCodebase(location=scan_data) c1 = virtual_codebase._get_or_create_resource( - 'some child', + "some child", parent=virtual_codebase.root, is_file=True, ) _c2 = virtual_codebase._get_or_create_resource( - 'some child2', + "some child2", parent=c1, is_file=False, ) results = list(virtual_codebase.walk(skip_root=True)) - expected = [('et131x.h', True), ('some child', True), - ('some child2', False)] + expected = [("et131x.h", True), ("some child", True), ("some child2", False)] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_walk_filtered_with_skip_root_and_single_file_with_children(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc("resource/virtual_codebase/et131x.h.json") virtual_codebase = VirtualCodebase(location=scan_data) c1 = virtual_codebase._get_or_create_resource( - 'some child', + "some child", parent=virtual_codebase.root, is_file=True, ) c2 = virtual_codebase._get_or_create_resource( - 'some child2', + "some child2", parent=c1, is_file=False, ) @@ -1085,197 +1045,181 @@ def test_virtual_codebase_walk_filtered_with_skip_root_and_single_file_with_chil c2.save(virtual_codebase) results = list(virtual_codebase.walk_filtered(skip_root=True)) - expected = [('et131x.h', True), ('some child', True)] + expected = [("et131x.h", True), ("some child", True)] assert [(r.name, r.is_file) for r in results] == expected c1.is_filtered = True c1.save(virtual_codebase) results = list(virtual_codebase.walk_filtered(skip_root=True)) - expected = [('et131x.h', True)] + expected = [("et131x.h", True)] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase__create_resource_can_add_child_to_file(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/et131x.h.json') + scan_data = self.get_test_loc("resource/virtual_codebase/et131x.h.json") virtual_codebase = VirtualCodebase(location=scan_data) virtual_codebase._get_or_create_resource( - 'some child', + "some child", virtual_codebase.root, is_file=True, ) results = list(virtual_codebase.walk()) - expected = [('et131x.h', True), ('some child', True)] + expected = [("et131x.h", True), ("some child", True)] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase__create_resource_can_add_child_to_dir(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/resource.json') + scan_data = self.get_test_loc("resource/virtual_codebase/resource.json") virtual_codebase = VirtualCodebase(location=scan_data) virtual_codebase._get_or_create_resource( - 'some child', + "some child", virtual_codebase.root, is_file=False, ) results = list(virtual_codebase.walk()) - expected = [('resource', False), ('some child', False)] + expected = [("resource", False), ("some child", False)] assert [(r.name, r.is_file) for r in results] == expected def test_virtual_codebase_get_resource(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/resource.json') + scan_data = self.get_test_loc("resource/virtual_codebase/resource.json") virtual_codebase = VirtualCodebase(location=scan_data) - assert not ( - virtual_codebase.root is virtual_codebase.get_resource('resource')) - assert virtual_codebase.get_resource( - 'resource') == virtual_codebase.root + assert not (virtual_codebase.root is virtual_codebase.get_resource("resource")) + assert virtual_codebase.get_resource("resource") == virtual_codebase.root def test_virtual_codebase_can_process_minimal_resources_without_info(self): - scan_data = self.get_test_loc('resource/virtual_codebase/noinfo.json') + scan_data = self.get_test_loc("resource/virtual_codebase/noinfo.json") codebase = VirtualCodebase(location=scan_data) expected = [ { - 'path': 'NOTICE', - 'type': 'file', - 'copyrights': [ + "path": "NOTICE", + "type": "file", + "copyrights": [ { - 'statements': ['Copyright (c) 2017 nexB Inc. and others.'], - 'holders': ['nexB Inc. and others.'], - 'authors': [], - 'start_line': 4, - 'end_line': 4, + "statements": ["Copyright (c) 2017 nexB Inc. and others."], + "holders": ["nexB Inc. and others."], + "authors": [], + "start_line": 4, + "end_line": 4, } ], - 'scan_errors': [], + "scan_errors": [], } ] assert [r.to_dict() for r in codebase.walk()] == expected def test_virtual_codebase_can_process_minimal_resources_with_only_path(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/only-path.json') + scan_data = self.get_test_loc("resource/virtual_codebase/only-path.json") codebase = VirtualCodebase(location=scan_data) expected = [ - {'path': 'samples', 'type': 'directory', 'scan_errors': []}, - {'path': 'samples/NOTICE', 'type': 'file', 'scan_errors': []}, + {"path": "samples", "type": "directory", "scan_errors": []}, + {"path": "samples/NOTICE", "type": "file", "scan_errors": []}, ] assert [r.to_dict() for r in codebase.walk()] == expected def test_VirtualCodebase_account_fingerprint_attribute(self): - test_file = self.get_test_loc( - "resource/virtual_codebase/fingerprint_attribute.json") + test_file = self.get_test_loc("resource/virtual_codebase/fingerprint_attribute.json") codebase = VirtualCodebase(test_file) - resources_fingerprint = [ - resource.fingerprint for resource in codebase.walk()] + resources_fingerprint = [resource.fingerprint for resource in codebase.walk()] assert "e30cf09443e7878dfed3288886e97542" in resources_fingerprint assert None in resources_fingerprint - assert codebase.get_resource( - 'apache_to_all_notable_lic_new') == codebase.root + assert codebase.get_resource("apache_to_all_notable_lic_new") == codebase.root assert resources_fingerprint.count(None) == 2 def test_VirtualCodebase_works_with_mapping_backed_codebase(self): - test_file = self.get_test_loc( - "resource/virtual_codebase/license-scan.json") + test_file = self.get_test_loc("resource/virtual_codebase/license-scan.json") codebase = VirtualCodebase(test_file) - resource = codebase.get_resource('scan-ref/license-notice.txt') + resource = codebase.get_resource("scan-ref/license-notice.txt") assert resource assert len(resource.license_expressions) == 1 class TestCodebaseLowestCommonParent(FileBasedTesting): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_lowest_common_parent_on_virtual_codebase(self): - scan_data = self.get_test_loc('resource/virtual_codebase/lcp.json') + scan_data = self.get_test_loc("resource/virtual_codebase/lcp.json") virtual_codebase = VirtualCodebase(location=scan_data) lcp = virtual_codebase.lowest_common_parent() - assert lcp.path == 'lcp/test1' - assert lcp.name == 'test1' + assert lcp.path == "lcp/test1" + assert lcp.name == "test1" def test_virtual_codebase_has_default_for_plugin_attributes(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/only-path.json') + scan_data = self.get_test_loc("resource/virtual_codebase/only-path.json") VirtualCodebase(location=scan_data) def test_lowest_common_parent_strip(self): - test_codebase = self.get_test_loc('resource/lcp/test1') + test_codebase = self.get_test_loc("resource/lcp/test1") codebase = Codebase(test_codebase) assert len(list(codebase.walk())) == 75 lcp = codebase.lowest_common_parent() - assert lcp.path == 'test1' - assert lcp.name == 'test1' - assert lcp.strip_root_path == '' - assert lcp.full_root_path.endswith('resource/lcp/test1') + assert lcp.path == "test1" + assert lcp.name == "test1" + assert lcp.strip_root_path == "" + assert lcp.full_root_path.endswith("resource/lcp/test1") def test_lowest_common_parent_2(self): - test_codebase = self.get_test_loc('resource/lcp/test1/zlib') + test_codebase = self.get_test_loc("resource/lcp/test1/zlib") codebase = Codebase(test_codebase) lcp = codebase.lowest_common_parent() - assert lcp.path == 'zlib' - assert lcp.name == 'zlib' - assert lcp.strip_root_path == '' - assert lcp.full_root_path.endswith('resource/lcp/test1/zlib') + assert lcp.path == "zlib" + assert lcp.name == "zlib" + assert lcp.strip_root_path == "" + assert lcp.full_root_path.endswith("resource/lcp/test1/zlib") def test_lowest_common_parent_3(self): - test_codebase = self.get_test_loc('resource/lcp/test1/simple') + test_codebase = self.get_test_loc("resource/lcp/test1/simple") codebase = Codebase(test_codebase) lcp = codebase.lowest_common_parent() - assert lcp.path == 'simple' - assert lcp.name == 'simple' - assert lcp.strip_root_path == '' + assert lcp.path == "simple" + assert lcp.name == "simple" + assert lcp.strip_root_path == "" def test_lowest_common_parent_deep(self): - test_codebase = self.get_test_loc('resource/lcp/test1/simple/org') + test_codebase = self.get_test_loc("resource/lcp/test1/simple/org") codebase = Codebase(test_codebase) lcp = codebase.lowest_common_parent() - assert lcp.path == 'org/jvnet/glassfish/comms/sipagent' - assert lcp.name == 'sipagent' - assert lcp.strip_root_path == 'jvnet/glassfish/comms/sipagent' + assert lcp.path == "org/jvnet/glassfish/comms/sipagent" + assert lcp.name == "sipagent" + assert lcp.strip_root_path == "jvnet/glassfish/comms/sipagent" assert lcp.full_root_path.endswith( - 'resource/lcp/test1/simple/org/jvnet/glassfish/comms/sipagent' + "resource/lcp/test1/simple/org/jvnet/glassfish/comms/sipagent" ) def test_lowest_common_parent_solo_file(self): - test_codebase = self.get_test_loc('resource/lcp/test1/screenshot.png') + test_codebase = self.get_test_loc("resource/lcp/test1/screenshot.png") codebase = Codebase(test_codebase) lcp = codebase.lowest_common_parent() - assert lcp.path == 'screenshot.png' - assert lcp.name == 'screenshot.png' - assert lcp.strip_root_path == '' - assert lcp.full_root_path.endswith('resource/lcp/test1/screenshot.png') + assert lcp.path == "screenshot.png" + assert lcp.name == "screenshot.png" + assert lcp.strip_root_path == "" + assert lcp.full_root_path.endswith("resource/lcp/test1/screenshot.png") class TestVirtualCodebaseCache(FileBasedTesting): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_virtual_codebase_cache_default(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/codebase-for-cache-tests.json') + scan_data = self.get_test_loc("resource/virtual_codebase/codebase-for-cache-tests.json") virtual_codebase = VirtualCodebase(location=scan_data) assert virtual_codebase.temp_dir assert virtual_codebase.cache_dir virtual_codebase.cache_dir root = virtual_codebase.root - cp = virtual_codebase._get_resource_cache_location( - root.path, create_dirs=False) + cp = virtual_codebase._get_resource_cache_location(root.path, create_dirs=False) assert not exists(cp) - cp = virtual_codebase._get_resource_cache_location( - root.path, create_dirs=True) + cp = virtual_codebase._get_resource_cache_location(root.path, create_dirs=True) assert not exists(cp) assert exists(parent_directory(cp)) - child = virtual_codebase._get_or_create_resource( - 'child', root, is_file=True) + child = virtual_codebase._get_or_create_resource("child", root, is_file=True) child.size = 12 virtual_codebase.save_resource(child) child_2 = virtual_codebase.get_resource(child.path) assert child_2 == child def test_virtual_codebase_cache_all_in_memory(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/codebase-for-cache-tests.json') + scan_data = self.get_test_loc("resource/virtual_codebase/codebase-for-cache-tests.json") virtual_codebase = VirtualCodebase(location=scan_data, max_in_memory=0) for path, res in virtual_codebase.resources_by_path.items(): assert res != Codebase.CACHED_RESOURCE @@ -1296,20 +1240,16 @@ def test_virtual_codebase_cache_all_in_memory(self): ) def test_virtual_codebase_cache_all_on_disk(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/codebase-for-cache-tests.json') - virtual_codebase = VirtualCodebase( - location=scan_data, max_in_memory=-1) + scan_data = self.get_test_loc("resource/virtual_codebase/codebase-for-cache-tests.json") + virtual_codebase = VirtualCodebase(location=scan_data, max_in_memory=-1) for path, res in virtual_codebase.resources_by_path.items(): - if res != Codebase.CACHED_RESOURCE: assert res.is_root else: res = virtual_codebase.get_resource(path) if res.is_root: - assert virtual_codebase.get_resource( - path) == virtual_codebase.root + assert virtual_codebase.get_resource(path) == virtual_codebase.root assert virtual_codebase._exists_in_memory(path) assert not virtual_codebase._exists_on_disk(path) else: @@ -1323,8 +1263,7 @@ def test_virtual_codebase_cache_all_on_disk(self): ) def test_virtual_codebase_cache_mixed_two_in_memory(self): - scan_data = self.get_test_loc( - 'resource/virtual_codebase/codebase-for-cache-tests.json') + scan_data = self.get_test_loc("resource/virtual_codebase/codebase-for-cache-tests.json") virtual_codebase = VirtualCodebase(location=scan_data, max_in_memory=2) counter = 0 @@ -1334,8 +1273,7 @@ def test_virtual_codebase_cache_mixed_two_in_memory(self): if res.is_root: assert ( - virtual_codebase.get_resource( - path).to_dict() == virtual_codebase.root.to_dict() + virtual_codebase.get_resource(path).to_dict() == virtual_codebase.root.to_dict() ) assert virtual_codebase._exists_in_memory(path) assert not virtual_codebase._exists_on_disk(path) @@ -1358,18 +1296,17 @@ def test_virtual_codebase_cache_mixed_two_in_memory(self): class TestVirtualCodebaseCreation(FileBasedTesting): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_VirtualCodebase_can_be_created_from_json_file(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/from_file.json') + test_file = self.get_test_loc("resource/virtual_codebase/from_file.json") codebase = VirtualCodebase(test_file) results = sorted(r.name for r in codebase.walk()) - expected = ['bar.svg', 'han'] + expected = ["bar.svg", "han"] assert results == expected def test_VirtualCodebase_can_be_created_from_json_string(self): - test_scan = ''' + test_scan = """ { "scancode_notice": "Generated with ScanCode and provided on an ....", "scancode_version": "2.9.7.post137.2e29fe3.dirty.20181120225811", @@ -1392,10 +1329,10 @@ def test_VirtualCodebase_can_be_created_from_json_string(self): } ] } - ''' + """ codebase = VirtualCodebase(test_scan) results = sorted(r.name for r in codebase.walk()) - expected = ['bar.svg', 'han'] + expected = ["bar.svg", "han"] assert results == expected def test_VirtualCodebase_can_be_created_from_dict(self): @@ -1413,94 +1350,84 @@ def test_VirtualCodebase_can_be_created_from_dict(self): codebase = VirtualCodebase(test_scan) results = sorted(r.name for r in codebase.walk()) - expected = ['bar.svg', 'han'] + expected = ["bar.svg", "han"] assert results == expected def test_VirtualCodebase_create_from_scan_with_no_root_and_missing_parents(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/samples-only-findings.json') + test_file = self.get_test_loc("resource/virtual_codebase/samples-only-findings.json") result_file = self.get_test_loc( - 'resource/virtual_codebase/samples-only-findings-expected.json' + "resource/virtual_codebase/samples-only-findings-expected.json" ) codebase = VirtualCodebase(test_file) expected_scan = json.load(open(result_file)) results = sorted(r.path for r in codebase.walk()) - expected = sorted(r.get('path') for r in expected_scan['files']) + expected = sorted(r.get("path") for r in expected_scan["files"]) assert results == expected def test_VirtualCodebase_check_that_already_existing_parent_is_updated_properly(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/root-is-not-first-resource.json') + test_file = self.get_test_loc("resource/virtual_codebase/root-is-not-first-resource.json") codebase = VirtualCodebase(test_file) - results = sorted((r.to_dict() for r in codebase.walk()), - key=lambda x: tuple(x.items())) + results = sorted((r.to_dict() for r in codebase.walk()), key=lambda x: tuple(x.items())) expected = [ - {'path': 'samples', 'type': 'directory', - 'summary': ['asd'], 'scan_errors': []}, - {'path': 'samples/NOTICE', 'type': 'file', - 'summary': [], 'scan_errors': []}, + {"path": "samples", "type": "directory", "summary": ["asd"], "scan_errors": []}, + {"path": "samples/NOTICE", "type": "file", "summary": [], "scan_errors": []}, ] assert results == expected def test_VirtualCodebase_create_from_multiple_scans(self): - test_file_1 = self.get_test_loc( - 'resource/virtual_codebase/combine-1.json') - test_file_2 = self.get_test_loc( - 'resource/virtual_codebase/combine-2.json') + test_file_1 = self.get_test_loc("resource/virtual_codebase/combine-1.json") + test_file_2 = self.get_test_loc("resource/virtual_codebase/combine-2.json") vinput = (test_file_1, test_file_2) codebase = VirtualCodebase(vinput) results = [r.to_dict(with_info=False) for r in codebase.walk()] expected_file = self.get_test_loc( - 'resource/virtual_codebase/combine-expected.json', + "resource/virtual_codebase/combine-expected.json", must_exist=False, ) check_against_expected_json_file(results, expected_file, regen=False) def test_VirtualCodebase_create_from_multiple_scans_shared_directory_names(self): test_file_1 = self.get_test_loc( - 'resource/virtual_codebase/combine-shared-directory-name-1.json' + "resource/virtual_codebase/combine-shared-directory-name-1.json" ) test_file_2 = self.get_test_loc( - 'resource/virtual_codebase/combine-shared-directory-name-2.json' + "resource/virtual_codebase/combine-shared-directory-name-2.json" ) vinput = (test_file_1, test_file_2) codebase = VirtualCodebase(location=vinput) results = [r.to_dict(with_info=False) for r in codebase.walk()] expected_file = self.get_test_loc( - 'resource/virtual_codebase/combine-shared-directory-name-expected.json', + "resource/virtual_codebase/combine-shared-directory-name-expected.json", must_exist=False, ) check_against_expected_json_file(results, expected_file, regen=False) def test_VirtualCodebase_compute_counts_with_full_root_info_one(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/full-root-info-one.json') + test_file = self.get_test_loc("resource/virtual_codebase/full-root-info-one.json") codebase = VirtualCodebase(test_file) resource = [r for r in codebase.walk() if r.is_file][0] - assert resource.path == 'home/foobar/scancode-toolkit/samples/README' + assert resource.path == "home/foobar/scancode-toolkit/samples/README" files_count, dirs_count, size_count = codebase.compute_counts() assert files_count == 1 assert dirs_count == 0 assert size_count == 236 def test_VirtualCodebase_with_full_root_info_one(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/full-root-info-one.json') + test_file = self.get_test_loc("resource/virtual_codebase/full-root-info-one.json") codebase = VirtualCodebase(test_file) results = [r.to_dict(with_info=True) for r in codebase.walk()] expected_file = self.get_test_loc( - 'resource/virtual_codebase/full-root-info-one-expected.json', must_exist=False + "resource/virtual_codebase/full-root-info-one-expected.json", must_exist=False ) check_against_expected_json_file(results, expected_file, regen=False) def test_VirtualCodebase_with_full_root_info_many(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/full-root-info-many.json') + test_file = self.get_test_loc("resource/virtual_codebase/full-root-info-many.json") codebase = VirtualCodebase(test_file) results = [r.to_dict(with_info=True) for r in codebase.walk()] expected_file = self.get_test_loc( - 'resource/virtual_codebase/full-root-info-many-expected.json', must_exist=False + "resource/virtual_codebase/full-root-info-many-expected.json", must_exist=False ) check_against_expected_json_file(results, expected_file, regen=False) @@ -1508,41 +1435,37 @@ def test_VirtualCodebase_can_compute_counts_with_null(self): # was failing with # size_count += child.size # TypeError: unsupported operand type(s) for +=: 'int' and 'NoneType' - test_file = self.get_test_loc( - 'resource/virtual_codebase/node-16-slim.json') + test_file = self.get_test_loc("resource/virtual_codebase/node-16-slim.json") codebase = VirtualCodebase(test_file) codebase.compute_counts() def test_VirtualCodebase_can_be_created_with_single_path(self): - test_file = self.get_test_loc( - 'resource/virtual_codebase/docker-hello-world.json') + test_file = self.get_test_loc("resource/virtual_codebase/docker-hello-world.json") VirtualCodebase(test_file) def test_VirtualCodebase_can_be_created_without_RecursionError(self): # was failing with RecursionError: maximum recursion depth exceeded - test_file = self.get_test_loc( - 'resource/virtual_codebase/zephyr-binary.json') + test_file = self.get_test_loc("resource/virtual_codebase/zephyr-binary.json") VirtualCodebase(test_file) def test_VirtualCodebase_can_be_created_with_repeated_root_directory(self): paths = [ - 'to', - 'to/to', - 'to/to/to', - 'to/to/to/to', + "to", + "to/to", + "to/to/to", + "to/to/to/to", ] - resources = [{'path': path} for path in paths] - vc = VirtualCodebase(location={'files': resources}) + resources = [{"path": path} for path in paths] + vc = VirtualCodebase(location={"files": resources}) walked_paths = [r.path for r in vc.walk()] assert paths == walked_paths class TestResource(FileBasedTesting): - test_data_dir = join(dirname(__file__), 'data') + test_data_dir = join(dirname(__file__), "data") def test_Resource_extracted_to_extracted_from(self): - test_file = self.get_test_loc( - 'resource/resource/test-extracted-from-to.json') + test_file = self.get_test_loc("resource/resource/test-extracted-from-to.json") codebase = VirtualCodebase(location=test_file) results = [] for r in codebase.walk(topdown=True): @@ -1555,75 +1478,74 @@ def test_Resource_extracted_to_extracted_from(self): expected = [ ( - 'test', + "test", None, None, ), ( - 'test/c', + "test/c", None, None, ), ( - 'test/foo.tar.gz', - 'test/foo.tar.gz-extract', + "test/foo.tar.gz", + "test/foo.tar.gz-extract", None, ), ( - 'test/foo.tar.gz-extract', + "test/foo.tar.gz-extract", None, - 'test/foo.tar.gz', + "test/foo.tar.gz", ), ( - 'test/foo.tar.gz-extract/foo', + "test/foo.tar.gz-extract/foo", None, - 'test/foo.tar.gz', + "test/foo.tar.gz", ), ( - 'test/foo.tar.gz-extract/foo/a', + "test/foo.tar.gz-extract/foo/a", None, - 'test/foo.tar.gz', + "test/foo.tar.gz", ), ( - 'test/foo.tar.gz-extract/foo/bar.tar.gz', - 'test/foo.tar.gz-extract/foo/bar.tar.gz-extract', - 'test/foo.tar.gz', + "test/foo.tar.gz-extract/foo/bar.tar.gz", + "test/foo.tar.gz-extract/foo/bar.tar.gz-extract", + "test/foo.tar.gz", ), ( - 'test/foo.tar.gz-extract/foo/bar.tar.gz-extract', + "test/foo.tar.gz-extract/foo/bar.tar.gz-extract", None, - 'test/foo.tar.gz-extract/foo/bar.tar.gz', + "test/foo.tar.gz-extract/foo/bar.tar.gz", ), ( - 'test/foo.tar.gz-extract/foo/bar.tar.gz-extract/bar', + "test/foo.tar.gz-extract/foo/bar.tar.gz-extract/bar", None, - 'test/foo.tar.gz-extract/foo/bar.tar.gz', + "test/foo.tar.gz-extract/foo/bar.tar.gz", ), ( - 'test/foo.tar.gz-extract/foo/bar.tar.gz-extract/bar/b', + "test/foo.tar.gz-extract/foo/bar.tar.gz-extract/bar/b", None, - 'test/foo.tar.gz-extract/foo/bar.tar.gz', + "test/foo.tar.gz-extract/foo/bar.tar.gz", ), ] assert results == expected def test_virtualcode_Resource_can_walk(self): - test_file = self.get_test_loc( - 'resource/resource/test-extracted-from-to.json') + test_file = self.get_test_loc("resource/resource/test-extracted-from-to.json") codebase = VirtualCodebase(location=test_file) results = [r.path for r in codebase.walk(topdown=True)] expected = [ - 'test', - 'test/c', - 'test/foo.tar.gz', - 'test/foo.tar.gz-extract', - 'test/foo.tar.gz-extract/foo', - 'test/foo.tar.gz-extract/foo/a', - 'test/foo.tar.gz-extract/foo/bar.tar.gz', - 'test/foo.tar.gz-extract/foo/bar.tar.gz-extract', - 'test/foo.tar.gz-extract/foo/bar.tar.gz-extract/bar', - 'test/foo.tar.gz-extract/foo/bar.tar.gz-extract/bar/b', + "test", + "test/c", + "test/foo.tar.gz", + "test/foo.tar.gz-extract", + "test/foo.tar.gz-extract/foo", + "test/foo.tar.gz-extract/foo/a", + "test/foo.tar.gz-extract/foo/bar.tar.gz", + "test/foo.tar.gz-extract/foo/bar.tar.gz-extract", + "test/foo.tar.gz-extract/foo/bar.tar.gz-extract/bar", + "test/foo.tar.gz-extract/foo/bar.tar.gz-extract/bar/b", ] assert results == expected diff --git a/tests/test_saneyaml.py b/tests/test_saneyaml.py index 2010bff2..fabcf88e 100644 --- a/tests/test_saneyaml.py +++ b/tests/test_saneyaml.py @@ -12,15 +12,15 @@ from commoncode.testcase import FileBasedTesting -TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), 'data') +TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data") class TestSaneyaml(FileBasedTesting): test_data_dir = TEST_DATA_DIR def test_load_with_and_without_tags(self): - test_file_with_tag = self.get_test_loc('saneyaml/metadata1') - test_file_without_tag = self.get_test_loc('saneyaml/metadata1.notag') - with_tags = saneyaml.load(open(test_file_with_tag, 'rb').read()) - without_tags = saneyaml.load(open(test_file_without_tag, 'rb').read()) + test_file_with_tag = self.get_test_loc("saneyaml/metadata1") + test_file_without_tag = self.get_test_loc("saneyaml/metadata1.notag") + with_tags = saneyaml.load(open(test_file_with_tag, "rb").read()) + without_tags = saneyaml.load(open(test_file_without_tag, "rb").read()) assert without_tags == with_tags diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py index 2eb6e558..95fcb9f5 100644 --- a/tests/test_skeleton_codestyle.py +++ b/tests/test_skeleton_codestyle.py @@ -7,9 +7,9 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import configparser import subprocess import unittest -import configparser class BaseTests(unittest.TestCase): diff --git a/tests/test_text.py b/tests/test_text.py index 4b08bc29..e151e886 100644 --- a/tests/test_text.py +++ b/tests/test_text.py @@ -13,7 +13,7 @@ def test_lines(): - t = '''This problem is. + t = """This problem is. It is therefore @@ -22,69 +22,75 @@ def test_lines(): However, I have -''' +""" assert len([p[1] for p in text.lines(t)]) == 5 - expected = ['This problem is.', 'It is therefore', - 'However,we', 'without introducing ..', 'However, I have'] + expected = [ + "This problem is.", + "It is therefore", + "However,we", + "without introducing ..", + "However, I have", + ] assert [p for p in text.lines(t)] == expected def test_foldcase(): - test = ' Fold THE cases of a text to lower casM' + test = " Fold THE cases of a text to lower casM" assert text.foldcase(test) == test.lower() def test_nopunctuation(): - test = '''This problem is about sequence-bunching, %^$^%**^&*©©^(*&(*()()_+)_!@@#:><>>?/./,.,';][{}{]just''' - expected = ['This', 'problem', 'is', 'about', - 'sequence', 'bunching', 'Â', 'Â', 'just'] + test = """This problem is about sequence-bunching, %^$^%**^&*©©^(*&(*()()_+)_!@@#:><>>?/./,.,';][{}{]just""" + expected = ["This", "problem", "is", "about", "sequence", "bunching", "Â", "Â", "just"] assert text.nopunctuation(test).split() == expected - test = 'This problem is about: sequence-bunching\n\n just \n' - expected = 'This problem is about sequence bunching just ' + test = "This problem is about: sequence-bunching\n\n just \n" + expected = "This problem is about sequence bunching just " assert text.nopunctuation(test) == expected def test_unixlinesep(): t = CR + LF + LF + CR + CR + LF assert text.unixlinesep(t) == LF + LF + LF + LF - assert text.unixlinesep(t, True) == ' ' + LF + LF + LF + ' ' + LF + assert text.unixlinesep(t, True) == " " + LF + LF + LF + " " + LF def test_nolinesep(): t = CR + LF + CR + CR + CR + LF - assert text.nolinesep(t) == ' ' + assert text.nolinesep(t) == " " def test_toascii(): - acc = u"ÀÁÂÃÄÅÇÈÉÊËÌÍÎÏÑÒÓÔÕÖØÙÚÛÜÝàáâãäåçèéêëìíîïñòóôõöøùúûüýÿẞß®©œŒØøÆæ₵₡￠¢Žž" - expected = r'AAAAAACEEEEIIIINOOOOOUUUUYaaaaaaceeeeiiiinooooouuuuyyZz' + acc = "ÀÁÂÃÄÅÇÈÉÊËÌÍÎÏÑÒÓÔÕÖØÙÚÛÜÝàáâãäåçèéêëìíîïñòóôõöøùúûüýÿẞß®©œŒØøÆæ₵₡￠¢Žž" + expected = r"AAAAAACEEEEIIIINOOOOOUUUUYaaaaaaceeeeiiiinooooouuuuyyZz" assert text.toascii(acc, translit=False) == expected - expected = r'AAAAAACEEEEIIIINOOOOOOUUUUYaaaaaaceeeeiiiinoooooouuuuyySsss(r)(c)oeOEOoAEae_CL/CC/Zz' + expected = ( + r"AAAAAACEEEEIIIINOOOOOOUUUUYaaaaaaceeeeiiiinoooooouuuuyySsss(r)(c)oeOEOoAEae_CL/CC/Zz" + ) assert text.toascii(acc, translit=True) == expected def test_toascii_works_with_empty_unicode_or_bytes(): - assert text.toascii(b'', translit=False) == u'' - assert text.toascii(u'', translit=True) == u'' - assert text.toascii(b'', translit=False) == u'' - assert text.toascii(u'', translit=True) == u'' + assert text.toascii(b"", translit=False) == "" + assert text.toascii("", translit=True) == "" + assert text.toascii(b"", translit=False) == "" + assert text.toascii("", translit=True) == "" def test_python_safe_name(): s = "not `\\a /`good` -safe name ??" - assert text.python_safe_name(s) == 'not___a___good___safe_name' + assert text.python_safe_name(s) == "not___a___good___safe_name" s1 = "string1++or+" s2 = "string1 +or " assert text.python_safe_name(s2) == text.python_safe_name(s1) def test_as_unicode(): - assert text.as_unicode('') == '' - assert isinstance(text.as_unicode(b'some bytes'), str) + assert text.as_unicode("") == "" + assert isinstance(text.as_unicode(b"some bytes"), str) assert text.as_unicode(None) == None try: - text.as_unicode(['foo']) - raise Exception('Exception should have been raised') + text.as_unicode(["foo"]) + raise Exception("Exception should have been raised") except AssertionError: pass diff --git a/tests/test_timeutils.py b/tests/test_timeutils.py index 1cd58f2f..568b6425 100644 --- a/tests/test_timeutils.py +++ b/tests/test_timeutils.py @@ -9,29 +9,28 @@ from datetime import datetime from commoncode.testcase import FileBasedTesting +from commoncode.timeutils import UTC from commoncode.timeutils import time2tstamp from commoncode.timeutils import tstamp2time -from commoncode.timeutils import UTC class TestTimeStamp(FileBasedTesting): - def test_time2tstamp_is_path_safe_and_file_is_writable(self): ts = time2tstamp() - tf = self.get_temp_file(extension='ext', dir_name=ts, file_name=ts) - fd = open(tf, 'w') - fd.write('a') + tf = self.get_temp_file(extension="ext", dir_name=ts, file_name=ts) + fd = open(tf, "w") + fd.write("a") fd.close() def test_time2tstamp_accepts_existing_datetimes(self): ts = time2tstamp() - tf = self.get_temp_file(extension='ext', dir_name=ts, file_name=ts) - fd = open(tf, 'w') - fd.write('a') + tf = self.get_temp_file(extension="ext", dir_name=ts, file_name=ts) + fd = open(tf, "w") + fd.write("a") fd.close() def test_time2tstamp_raises_on_non_datetime(self): - self.assertRaises(AttributeError, time2tstamp, 'some') + self.assertRaises(AttributeError, time2tstamp, "some") self.assertRaises(AttributeError, time2tstamp, 1) def test_time2tstamp_tstamp2time_is_idempotent(self): @@ -42,54 +41,154 @@ def test_time2tstamp_tstamp2time_is_idempotent(self): def test_tstamp2time_format(self): import re + ts = time2tstamp() - pat = r'^20\d\d-[0-1][0-9]-[0-3]\dT[0-2]\d[0-6]\d[0-6]\d.\d\d\d\d\d\d$' + pat = r"^20\d\d-[0-1][0-9]-[0-3]\dT[0-2]\d[0-6]\d[0-6]\d.\d\d\d\d\d\d$" assert re.match(pat, ts) def test_tstamp2time(self): - dt_from_ts = tstamp2time('2010-11-12T131415.000016') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, - second=15, microsecond=16, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("2010-11-12T131415.000016") + assert ( + datetime( + year=2010, + month=11, + day=12, + hour=13, + minute=14, + second=15, + microsecond=16, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time2(self): - dt_from_ts = tstamp2time('20101112T131415.000016') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, - second=15, microsecond=16, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("20101112T131415.000016") + assert ( + datetime( + year=2010, + month=11, + day=12, + hour=13, + minute=14, + second=15, + microsecond=16, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time3(self): - dt_from_ts = tstamp2time('20101112T131415.000016Z') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, - second=15, microsecond=16, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("20101112T131415.000016Z") + assert ( + datetime( + year=2010, + month=11, + day=12, + hour=13, + minute=14, + second=15, + microsecond=16, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time4(self): - dt_from_ts = tstamp2time('2010-11-12T131415') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, - second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("2010-11-12T131415") + assert ( + datetime( + year=2010, + month=11, + day=12, + hour=13, + minute=14, + second=15, + microsecond=0, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time5(self): - dt_from_ts = tstamp2time('2010-11-12T13:14:15') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, - second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("2010-11-12T13:14:15") + assert ( + datetime( + year=2010, + month=11, + day=12, + hour=13, + minute=14, + second=15, + microsecond=0, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time6(self): - dt_from_ts = tstamp2time('20101112T13:14:15') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, - second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("20101112T13:14:15") + assert ( + datetime( + year=2010, + month=11, + day=12, + hour=13, + minute=14, + second=15, + microsecond=0, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time7(self): - dt_from_ts = tstamp2time('20101112T13:14:15Z') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, - second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("20101112T13:14:15Z") + assert ( + datetime( + year=2010, + month=11, + day=12, + hour=13, + minute=14, + second=15, + microsecond=0, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time8(self): - dt_from_ts = tstamp2time('20101112T13:14:15Z') - assert datetime(year=2010, month=11, day=12, hour=13, minute=14, - second=15, microsecond=0, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("20101112T13:14:15Z") + assert ( + datetime( + year=2010, + month=11, + day=12, + hour=13, + minute=14, + second=15, + microsecond=0, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time9(self): - dt_from_ts = tstamp2time('2010-06-30T21:26:40.000Z') - assert datetime(year=2010, month=6, day=30, hour=21, minute=26, - second=40, microsecond=0, tzinfo=UTC()) == dt_from_ts + dt_from_ts = tstamp2time("2010-06-30T21:26:40.000Z") + assert ( + datetime( + year=2010, + month=6, + day=30, + hour=21, + minute=26, + second=40, + microsecond=0, + tzinfo=UTC(), + ) + == dt_from_ts + ) def test_tstamp2time_raise(self): - self.assertRaises(ValueError, tstamp2time, '201011A12T13:14:15Z') + self.assertRaises(ValueError, tstamp2time, "201011A12T13:14:15Z") diff --git a/tests/test_urn.py b/tests/test_urn.py index 1a85c786..6cc6ee99 100644 --- a/tests/test_urn.py +++ b/tests/test_urn.py @@ -12,135 +12,131 @@ class URNTestCase(unittest.TestCase): - def test_encode_license(self): - u1 = urn.encode('license', key='somekey') - assert u1 == 'urn:dje:license:somekey' + u1 = urn.encode("license", key="somekey") + assert u1 == "urn:dje:license:somekey" def test_encode_owner(self): - u1 = urn.encode('owner', name='somekey') - assert u1 == 'urn:dje:owner:somekey' + u1 = urn.encode("owner", name="somekey") + assert u1 == "urn:dje:owner:somekey" def test_encode_component(self): - u1 = urn.encode('component', name='name', version='version') - assert u1 == 'urn:dje:component:name:version' + u1 = urn.encode("component", name="name", version="version") + assert u1 == "urn:dje:component:name:version" def test_encode_component_no_version(self): - u1 = urn.encode('component', name='name', version='') - assert u1 == 'urn:dje:component:name:' + u1 = urn.encode("component", name="name", version="") + assert u1 == "urn:dje:component:name:" def test_encode_license_with_extra_fields_are_ignored(self): - u1 = urn.encode('license', key='somekey', junk='somejunk') - assert u1 == 'urn:dje:license:somekey' + u1 = urn.encode("license", key="somekey", junk="somejunk") + assert u1 == "urn:dje:license:somekey" def test_encode_missing_field_raise_keyerror(self): with self.assertRaises(KeyError): - urn.encode('license') + urn.encode("license") def test_encode_missing_field_component_raise_keyerror(self): with self.assertRaises(KeyError): - urn.encode('component', name='this') + urn.encode("component", name="this") def test_encode_unknown_object_type_raise_keyerror(self): with self.assertRaises(KeyError): - urn.encode('some', key='somekey') + urn.encode("some", key="somekey") def test_encode_component_with_spaces_are_properly_quoted(self): - u1 = urn.encode('component', name='name space', - version='version space') - assert u1 == 'urn:dje:component:name+space:version+space' + u1 = urn.encode("component", name="name space", version="version space") + assert u1 == "urn:dje:component:name+space:version+space" def test_encode_leading_and_trailing_spaces_are_trimmed_and_ignored(self): - u1 = urn.encode(' component ', name=' name space ', - version=''' version space ''') - assert u1 == 'urn:dje:component:name+space:version+space' + u1 = urn.encode(" component ", name=" name space ", version=""" version space """) + assert u1 == "urn:dje:component:name+space:version+space" def test_encode_component_with_semicolon_are_properly_quoted(self): - u1 = urn.encode('component', name='name:', version=':version') - assert u1 == 'urn:dje:component:name%3A:%3Aversion' + u1 = urn.encode("component", name="name:", version=":version") + assert u1 == "urn:dje:component:name%3A:%3Aversion" def test_encode_component_with_plus_are_properly_quoted(self): - u1 = urn.encode('component', name='name+', version='version+') - assert u1 == 'urn:dje:component:name%2B:version%2B' + u1 = urn.encode("component", name="name+", version="version+") + assert u1 == "urn:dje:component:name%2B:version%2B" def test_encode_component_with_percent_are_properly_quoted(self): - u1 = urn.encode('component', name='name%', version='version%') - assert u1 == 'urn:dje:component:name%25:version%25' + u1 = urn.encode("component", name="name%", version="version%") + assert u1 == "urn:dje:component:name%25:version%25" def test_encode_object_type_case_is_not_significant(self): - u1 = urn.encode('license', key='key') - u2 = urn.encode('lICENSe', key='key') + u1 = urn.encode("license", key="key") + u2 = urn.encode("lICENSe", key="key") assert u2 == u1 def test_decode_component(self): - u = 'urn:dje:component:name:version' - parsed = ('component', {'name': 'name', 'version': 'version'}) + u = "urn:dje:component:name:version" + parsed = ("component", {"name": "name", "version": "version"}) assert urn.decode(u) == parsed def test_decode_license(self): - u = 'urn:dje:license:lic' - parsed = ('license', {'key': 'lic'}) + u = "urn:dje:license:lic" + parsed = ("license", {"key": "lic"}) assert urn.decode(u) == parsed def test_decode_org(self): - u = 'urn:dje:owner:name' - parsed = ('owner', {'name': 'name'}) + u = "urn:dje:owner:name" + parsed = ("owner", {"name": "name"}) assert urn.decode(u) == parsed def test_decode_build_is_idempotent(self): - u1 = urn.encode('component', owner__name='org%', name='name%', - version='version%') + u1 = urn.encode("component", owner__name="org%", name="name%", version="version%") m, f = urn.decode(u1) u3 = urn.encode(m, **f) assert u3 == u1 def test_decode_raise_exception_if_incorrect_prefix(self): with self.assertRaises(urn.URNValidationError): - urn.decode('arn:dje:a:a') + urn.decode("arn:dje:a:a") def test_decode_raise_exception_if_incorrect_ns(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:x:x:x') + urn.decode("urn:x:x:x") def test_decode_raise_exception_if_incorrect_prefix_or_ns(self): with self.assertRaises(urn.URNValidationError): - urn.decode('x:x:x:x') + urn.decode("x:x:x:x") def test_decode_raise_exception_if_too_short_license(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:dje:license') + urn.decode("urn:dje:license") def test_decode_raise_exception_if_too_short_component(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:dje:component') + urn.decode("urn:dje:component") def test_decode_raise_exception_if_too_long(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:dje:owner:o:n') + urn.decode("urn:dje:owner:o:n") def test_decode_raise_exception_if_too_long1(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:dje:component:o:n:v:junk') + urn.decode("urn:dje:component:o:n:v:junk") def test_decode_raise_exception_if_too_long2(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:dje:owner:org:junk') + urn.decode("urn:dje:owner:org:junk") def test_decode_raise_exception_if_too_long3(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:dje:license:key:junk') + urn.decode("urn:dje:license:key:junk") def test_decode_raise_exception_if_unknown_object_type(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:dje:marshmallows:dsds') + urn.decode("urn:dje:marshmallows:dsds") def test_decode_raise_exception_if_missing_object_type(self): with self.assertRaises(urn.URNValidationError): - urn.decode('urn:dje::dsds') + urn.decode("urn:dje::dsds") def test_encode_decode_is_idempotent(self): - object_type = 'component' - fields = {'name': 'SIP Servlets (MSS)', 'version': 'v 1.4.0.FINAL'} - encoded = 'urn:dje:component:SIP+Servlets+%28MSS%29:v+1.4.0.FINAL' + object_type = "component" + fields = {"name": "SIP Servlets (MSS)", "version": "v 1.4.0.FINAL"} + encoded = "urn:dje:component:SIP+Servlets+%28MSS%29:v+1.4.0.FINAL" assert urn.encode(object_type, **fields) == encoded assert urn.decode(encoded) == (object_type, fields) diff --git a/tests/test_version.py b/tests/test_version.py index 9f54b841..6dafa30f 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -12,112 +12,111 @@ class TestVersionHint(unittest.TestCase): - def test_version_hint(self): data = { - '/xmlgraphics/fop/source/fop-1.0-src.zip': '1.0', - '/xml/xindice/xml-xindice-1.2m1-src.zip': '1.2m1', - '/xmlgraphics/fop/binaries/fop-0.94-bin-jdk1.3.tar.gz': '0.94', - '/xmlgraphics/batik/batik-src-1.7beta1.zip': '1.7beta1', - '/xmlgraphics/batik/batik-1.7-jre13.zip': '1.7', - '/xmlbeans/source/xmlbeans-2.3.0-src.tgz': '2.3.0', - '/xml/xindice/source/xml-xindice-1.2m1-src.tar.gz': '1.2m1', - '/xml/xerces-p/binaries/XML-Xerces-2.3.0-4-win32.zip': '2.3.0-4', - '/xml/xerces-p/source/XML-Xerces-2.3.0-3.tar.gz': '2.3.0-3', - '/xml/xalan-j/source/xalan-j_2_7_0-src-2jars.tar.gz': '2_7_0', - '/xml/security/java-library/xml-security-src-1_0_5D2.zip': '1_0_5D2', - '/xml/commons/binaries/xml-commons-external-1.4.01-bin.zip': '1.4.01', - '/xml/commons/xml-commons-1.0.b2.zip': '1.0.b2', - '/xml/cocoon/3.0/cocoon-all-3.0.0-alpha-1-dist.tar.gz': '3.0.0-alpha-1', - '/xerces/j/source/Xerces-J-tools.2.10.0-xml-schema-1.1-beta.tar.gz': '2.10.0', - '/xerces/c/3/binaries/xerces-c-3.1.1-x86_64-solaris-cc-5.10.tar.gz': '3.1.1', - '/xerces/c/3/binaries/xerces-c-3.1.1-x86_64-windows-vc-8.0.zip': '3.1.1', - '/xerces/c/2/binaries/xerces-c_2_8_0-x86-windows-vc_7_1.zip': '2_8_0', - '/ws/woden/1.0M8/apache-woden-src-1.0M8.tar.gz': '1.0M8', - '/ws/scout/0_7rc1/source/scout-0.7rc1-src.zip': '0.7rc1', - '/ws/juddi/3_0/juddi-portal-bundle-3.0.0.rc1.zip': '3.0.0.rc1', - '/ws/juddi/3_0/juddi-portal-bundle-3.0.0.beta.zip': '3.0.0.beta', - '/ws/juddi/2_0RC7/juddi-tomcat-2.0rc7.zip': '2.0rc7', - '/ws/axis2/tools/1_4_1/axis2-wsdl2code-maven-plugin-1.4.1.jar': '1.4.1', - '/ws/axis/1_4/axis-src-1_4.zip': '1_4', - '/ws/axis-c/source/win32/axis-c-1.6b-Win32-trace-src.zip': '1.6b', - '/tuscany/java/sca/2.0-M5/apache-tuscany-sca-all-2.0-M5-src.tar.gz': '2.0-M5', - '/turbine/turbine-2.3.3-rc1/source/turbine-2.3.3-RC1-src.zip': '2.3.3-RC1', - '/tomcat/tomcat-connectors/jk/binaries/win64/jk-1.2.30/ia64/symbols-1.2.30.zip': '1.2.30', - '/tomcat/tomcat-7/v7.0.0-beta/bin/apache-tomcat-7.0.0-windows-i64.zip': '7.0.0', - '/tomcat/tomcat-4/v4.1.40/bin/apache-tomcat-4.1.40-LE-jdk14.exe': '4.1.40', - '/tapestry/tapestry-src-5.1.0.5.tar.gz': '5.1.0.5', - '/spamassassin/source/Mail-SpamAssassin-rules-3.3.0.r901671.tgz': '3.3.0.r901671', - '/spamassassin/Mail-SpamAssassin-rules-3.3.1.r923257.tgz': '3.3.1.r923257', - '/shindig/1.1-BETA5-incubating/shindig-1.1-BETA5-incubating-source.zip': '1.1-BETA5', - '/servicemix/nmr/1.0.0-m3/apache-servicemix-nmr-1.0.0-m3-src.tar.gz': '1.0.0-m3', - '/qpid/0.6/qpid-dotnet-0-10-0.6.zip': '0.6', - '/openjpa/2.0.0-beta/apache-openjpa-2.0.0-beta-binary.zip': '2.0.0-beta', - '/myfaces/source/portlet-bridge-2.0.0-alpha-2-src-all.tar.gz': '2.0.0-alpha-2', - '/myfaces/source/myfaces-extval20-2.0.3-src.tar.gz': '2.0.3', - '/harmony/milestones/6.0/debian/amd64/harmony-6.0-classlib_0.0r946981-1_amd64.deb': '6.0', - '/geronimo/eclipse/updates/plugins/org.apache.geronimo.st.v21.ui_2.1.1.jar': '2.1.1', - '/directory/studio/update/1.x/plugins/org.apache.directory.studio.aciitemeditor_1.5.2.v20091211.jar': '1.5.2.v20091211', - '/db/torque/torque-3.3/source/torque-gen-3.3-RC3-src.zip': '3.3-RC3', - '/cayenne/cayenne-3.0B1.tar.gz': '3.0B1', - '/cayenne/cayenne-3.0M4-macosx.dmg': '3.0M4', - '/xmlgraphics/batik/batik-docs-current.zip': 'current', - '/xmlgraphics/batik/batik-docs-previous.zip': 'previous', - '/poi/dev/bin/poi-bin-3.7-beta1-20100620.zip': '3.7-beta1-20100620', - '/excalibur/avalon-logkit/source/excalibur-logkit-2.0.dev-0-src.zip': '2.0.dev-0', - '/db/derby/db-derby-10.4.2.0/derby_core_plugin_10.4.2.zip': '10.4.2', - '/httpd/modpython/win/2.7.1/mp152dll.zip': '2.7.1', - '/perl/mod_perl-1.31/apaci/mod_perl.config.sh': '1.31', - '/xml/xerces-j/old_xerces2/Xerces-J-bin.2.0.0.alpha.zip': '2.0.0.alpha', - '/xml/xerces-p/archives/XML-Xerces-1.7.0_0.tar.gz': '1.7.0_0', - '/httpd/docs/tools-2004-05-04.zip': '2004-05-04', - '/ws/axis2/c/M0_5/axis2c-src-M0.5.tar.gz': 'M0.5', - '/jakarta/poi/dev/src/jakarta-poi-1.8.0-dev-src.zip': '1.8.0-dev', - '/tapestry/tapestry-4.0-beta-8.zip': '4.0-beta-8', - '/openejb/3.0-beta-1/openejb-3.0-beta-1.zip': '3.0-beta-1', - '/tapestry/tapestry-4.0-rc-1.zip': '4.0-rc-1', - '/jakarta/tapestry/source/3.0-rc-3/Tapestry-3.0-rc-3-src.zip': '3.0-rc-3', - '/jakarta/lucene/binaries/lucene-1.3-final.tar.gz': '1.3-final', - '/jakarta/tapestry/binaries/3.0-beta-1a/Tapestry-3.0-beta-1a-bin.zip': '3.0-beta-1a', - '/poi/release/bin/poi-bin-3.0-FINAL-20070503.tar.gz': '3.0-FINAL-20070503', - '/harmony/milestones/M4/apache-harmony-hdk-r603534-linux-x86-32-libstdc++v6-snapshot.tar.gz': 'r603534', - '/ant/antidote/antidote-20050330.tar.bz2': '20050330', - '/apr/not-released/apr_20020725223645.tar.gz': '20020725223645', - '/ibatis/source/ibatis.net/src-revision-709676.zip': 'revision-709676', - '/ws/axis-c/source/win32/axis-c-src-1-2-win32.zip': '1-2', - '/jakarta/slide/most-recent-2.0rc1-binaries/jakarta-slide 2.0rc1 jakarta-tomcat-4.1.30.zip': '2.0rc1', - '/httpd/modpython/win/3.0.1/python2.2.1-apache2.0.43.zip': '2.2.1', - '/ant/ivyde/updatesite/features/org.apache.ivy.feature_2.1.0.cr1_20090319213629.jar': '2.1.0.cr1_20090319213629', - '/jakarta/poi/dev/bin/poi-2.0-pre1-20030517.jar': '2.0-pre1-20030517', - '/jakarta/poi/release/bin/jakarta-poi-1.5.0-FINAL-bin.zip': '1.5.0-FINAL', - '/jakarta/poi/release/bin/poi-bin-2.0-final-20040126.zip': '2.0-final-20040126', - '/activemq/apache-activemq/5.0.0/apache-activemq-5.0.0-sources.jar': '5.0.0', - '/turbine/turbine-2.2/source/jakarta-turbine-2.2-B1.tar.gz': '2.2-B1', - '/ant/ivyde/updatesite/features/org.apache.ivy.feature_2.0.0.cr1.jar': '2.0.0.cr1', - '/ant/ivyde/updatesite/features/org.apache.ivy.feature_2.0.0.final_20090108225011.jar': '2.0.0.final_20090108225011', - '/ws/axis/1_2RC3/axis-src-1_2RC3.zip': '1_2RC3', - '/commons/lang/old/v1.0-b1.1/commons-lang-1.0-b1.1.zip': '1.0-b1.1', - '/commons/net/binaries/commons-net-1.2.0-release.tar.gz': '1.2.0-release', - '/ant/ivyde/2.0.0.final/apache-ivyde-2.0.0.final-200907011148-RELEASE.tgz': '2.0.0.final-200907011148-RELEASE', - '/geronimo/eclipse/updates/plugins/org.apache.geronimo.jetty.j2ee.server.v11_1.0.0.jar': 'v11_1.0.0', - '/jakarta/cactus/binaries/jakarta-cactus-13-1.7.1-fixed.zip': '1.7.1-fixed', - '/jakarta/jakarta-turbine-maven/maven/jars/maven-1.0-b5-dev.20020731.085427.jar': '1.0-b5-dev.20020731.085427', - '/xml/xalan-j/source/xalan-j_2_5_D1-src.tar.gz': '2_5_D1', - '/ws/woden/IBuilds/I20051002_1145/woden-I20051002_1145.tar.bz2': 'I20051002_1145', - '/commons/beanutils/source/commons-beanutils-1.8.0-BETA-src.tar.gz': '1.8.0-BETA', - '/cocoon/BINARIES/cocoon-2.0.3-vm14-bin.tar.gz': '2.0.3-vm14', - '/felix/xliff_filters_v1_2_7_unix.jar': 'v1_2_7', - '/excalibur/releases/200702/excalibur-javadoc-r508111-15022007.tar.gz': 'r508111-15022007', - '/geronimo/eclipse/updates/features/org.apache.geronimo.v20.feature_2.0.0.jar': 'v20.feature_2.0.0', - '/geronimo/2.1.6/axis2-jaxws-1.3-G20090406.jar': '1.3-G20090406', - '/cassandra/debian/pool/main/c/cassandra/cassandra_0.4.0~beta1-1.diff.gz': '0.4.0~beta1-1', - '/ha-api-3.1.6.jar': '3.1.6', - 'ha-api-3.1.6.jar': '3.1.6' + "/xmlgraphics/fop/source/fop-1.0-src.zip": "1.0", + "/xml/xindice/xml-xindice-1.2m1-src.zip": "1.2m1", + "/xmlgraphics/fop/binaries/fop-0.94-bin-jdk1.3.tar.gz": "0.94", + "/xmlgraphics/batik/batik-src-1.7beta1.zip": "1.7beta1", + "/xmlgraphics/batik/batik-1.7-jre13.zip": "1.7", + "/xmlbeans/source/xmlbeans-2.3.0-src.tgz": "2.3.0", + "/xml/xindice/source/xml-xindice-1.2m1-src.tar.gz": "1.2m1", + "/xml/xerces-p/binaries/XML-Xerces-2.3.0-4-win32.zip": "2.3.0-4", + "/xml/xerces-p/source/XML-Xerces-2.3.0-3.tar.gz": "2.3.0-3", + "/xml/xalan-j/source/xalan-j_2_7_0-src-2jars.tar.gz": "2_7_0", + "/xml/security/java-library/xml-security-src-1_0_5D2.zip": "1_0_5D2", + "/xml/commons/binaries/xml-commons-external-1.4.01-bin.zip": "1.4.01", + "/xml/commons/xml-commons-1.0.b2.zip": "1.0.b2", + "/xml/cocoon/3.0/cocoon-all-3.0.0-alpha-1-dist.tar.gz": "3.0.0-alpha-1", + "/xerces/j/source/Xerces-J-tools.2.10.0-xml-schema-1.1-beta.tar.gz": "2.10.0", + "/xerces/c/3/binaries/xerces-c-3.1.1-x86_64-solaris-cc-5.10.tar.gz": "3.1.1", + "/xerces/c/3/binaries/xerces-c-3.1.1-x86_64-windows-vc-8.0.zip": "3.1.1", + "/xerces/c/2/binaries/xerces-c_2_8_0-x86-windows-vc_7_1.zip": "2_8_0", + "/ws/woden/1.0M8/apache-woden-src-1.0M8.tar.gz": "1.0M8", + "/ws/scout/0_7rc1/source/scout-0.7rc1-src.zip": "0.7rc1", + "/ws/juddi/3_0/juddi-portal-bundle-3.0.0.rc1.zip": "3.0.0.rc1", + "/ws/juddi/3_0/juddi-portal-bundle-3.0.0.beta.zip": "3.0.0.beta", + "/ws/juddi/2_0RC7/juddi-tomcat-2.0rc7.zip": "2.0rc7", + "/ws/axis2/tools/1_4_1/axis2-wsdl2code-maven-plugin-1.4.1.jar": "1.4.1", + "/ws/axis/1_4/axis-src-1_4.zip": "1_4", + "/ws/axis-c/source/win32/axis-c-1.6b-Win32-trace-src.zip": "1.6b", + "/tuscany/java/sca/2.0-M5/apache-tuscany-sca-all-2.0-M5-src.tar.gz": "2.0-M5", + "/turbine/turbine-2.3.3-rc1/source/turbine-2.3.3-RC1-src.zip": "2.3.3-RC1", + "/tomcat/tomcat-connectors/jk/binaries/win64/jk-1.2.30/ia64/symbols-1.2.30.zip": "1.2.30", + "/tomcat/tomcat-7/v7.0.0-beta/bin/apache-tomcat-7.0.0-windows-i64.zip": "7.0.0", + "/tomcat/tomcat-4/v4.1.40/bin/apache-tomcat-4.1.40-LE-jdk14.exe": "4.1.40", + "/tapestry/tapestry-src-5.1.0.5.tar.gz": "5.1.0.5", + "/spamassassin/source/Mail-SpamAssassin-rules-3.3.0.r901671.tgz": "3.3.0.r901671", + "/spamassassin/Mail-SpamAssassin-rules-3.3.1.r923257.tgz": "3.3.1.r923257", + "/shindig/1.1-BETA5-incubating/shindig-1.1-BETA5-incubating-source.zip": "1.1-BETA5", + "/servicemix/nmr/1.0.0-m3/apache-servicemix-nmr-1.0.0-m3-src.tar.gz": "1.0.0-m3", + "/qpid/0.6/qpid-dotnet-0-10-0.6.zip": "0.6", + "/openjpa/2.0.0-beta/apache-openjpa-2.0.0-beta-binary.zip": "2.0.0-beta", + "/myfaces/source/portlet-bridge-2.0.0-alpha-2-src-all.tar.gz": "2.0.0-alpha-2", + "/myfaces/source/myfaces-extval20-2.0.3-src.tar.gz": "2.0.3", + "/harmony/milestones/6.0/debian/amd64/harmony-6.0-classlib_0.0r946981-1_amd64.deb": "6.0", + "/geronimo/eclipse/updates/plugins/org.apache.geronimo.st.v21.ui_2.1.1.jar": "2.1.1", + "/directory/studio/update/1.x/plugins/org.apache.directory.studio.aciitemeditor_1.5.2.v20091211.jar": "1.5.2.v20091211", + "/db/torque/torque-3.3/source/torque-gen-3.3-RC3-src.zip": "3.3-RC3", + "/cayenne/cayenne-3.0B1.tar.gz": "3.0B1", + "/cayenne/cayenne-3.0M4-macosx.dmg": "3.0M4", + "/xmlgraphics/batik/batik-docs-current.zip": "current", + "/xmlgraphics/batik/batik-docs-previous.zip": "previous", + "/poi/dev/bin/poi-bin-3.7-beta1-20100620.zip": "3.7-beta1-20100620", + "/excalibur/avalon-logkit/source/excalibur-logkit-2.0.dev-0-src.zip": "2.0.dev-0", + "/db/derby/db-derby-10.4.2.0/derby_core_plugin_10.4.2.zip": "10.4.2", + "/httpd/modpython/win/2.7.1/mp152dll.zip": "2.7.1", + "/perl/mod_perl-1.31/apaci/mod_perl.config.sh": "1.31", + "/xml/xerces-j/old_xerces2/Xerces-J-bin.2.0.0.alpha.zip": "2.0.0.alpha", + "/xml/xerces-p/archives/XML-Xerces-1.7.0_0.tar.gz": "1.7.0_0", + "/httpd/docs/tools-2004-05-04.zip": "2004-05-04", + "/ws/axis2/c/M0_5/axis2c-src-M0.5.tar.gz": "M0.5", + "/jakarta/poi/dev/src/jakarta-poi-1.8.0-dev-src.zip": "1.8.0-dev", + "/tapestry/tapestry-4.0-beta-8.zip": "4.0-beta-8", + "/openejb/3.0-beta-1/openejb-3.0-beta-1.zip": "3.0-beta-1", + "/tapestry/tapestry-4.0-rc-1.zip": "4.0-rc-1", + "/jakarta/tapestry/source/3.0-rc-3/Tapestry-3.0-rc-3-src.zip": "3.0-rc-3", + "/jakarta/lucene/binaries/lucene-1.3-final.tar.gz": "1.3-final", + "/jakarta/tapestry/binaries/3.0-beta-1a/Tapestry-3.0-beta-1a-bin.zip": "3.0-beta-1a", + "/poi/release/bin/poi-bin-3.0-FINAL-20070503.tar.gz": "3.0-FINAL-20070503", + "/harmony/milestones/M4/apache-harmony-hdk-r603534-linux-x86-32-libstdc++v6-snapshot.tar.gz": "r603534", + "/ant/antidote/antidote-20050330.tar.bz2": "20050330", + "/apr/not-released/apr_20020725223645.tar.gz": "20020725223645", + "/ibatis/source/ibatis.net/src-revision-709676.zip": "revision-709676", + "/ws/axis-c/source/win32/axis-c-src-1-2-win32.zip": "1-2", + "/jakarta/slide/most-recent-2.0rc1-binaries/jakarta-slide 2.0rc1 jakarta-tomcat-4.1.30.zip": "2.0rc1", + "/httpd/modpython/win/3.0.1/python2.2.1-apache2.0.43.zip": "2.2.1", + "/ant/ivyde/updatesite/features/org.apache.ivy.feature_2.1.0.cr1_20090319213629.jar": "2.1.0.cr1_20090319213629", + "/jakarta/poi/dev/bin/poi-2.0-pre1-20030517.jar": "2.0-pre1-20030517", + "/jakarta/poi/release/bin/jakarta-poi-1.5.0-FINAL-bin.zip": "1.5.0-FINAL", + "/jakarta/poi/release/bin/poi-bin-2.0-final-20040126.zip": "2.0-final-20040126", + "/activemq/apache-activemq/5.0.0/apache-activemq-5.0.0-sources.jar": "5.0.0", + "/turbine/turbine-2.2/source/jakarta-turbine-2.2-B1.tar.gz": "2.2-B1", + "/ant/ivyde/updatesite/features/org.apache.ivy.feature_2.0.0.cr1.jar": "2.0.0.cr1", + "/ant/ivyde/updatesite/features/org.apache.ivy.feature_2.0.0.final_20090108225011.jar": "2.0.0.final_20090108225011", + "/ws/axis/1_2RC3/axis-src-1_2RC3.zip": "1_2RC3", + "/commons/lang/old/v1.0-b1.1/commons-lang-1.0-b1.1.zip": "1.0-b1.1", + "/commons/net/binaries/commons-net-1.2.0-release.tar.gz": "1.2.0-release", + "/ant/ivyde/2.0.0.final/apache-ivyde-2.0.0.final-200907011148-RELEASE.tgz": "2.0.0.final-200907011148-RELEASE", + "/geronimo/eclipse/updates/plugins/org.apache.geronimo.jetty.j2ee.server.v11_1.0.0.jar": "v11_1.0.0", + "/jakarta/cactus/binaries/jakarta-cactus-13-1.7.1-fixed.zip": "1.7.1-fixed", + "/jakarta/jakarta-turbine-maven/maven/jars/maven-1.0-b5-dev.20020731.085427.jar": "1.0-b5-dev.20020731.085427", + "/xml/xalan-j/source/xalan-j_2_5_D1-src.tar.gz": "2_5_D1", + "/ws/woden/IBuilds/I20051002_1145/woden-I20051002_1145.tar.bz2": "I20051002_1145", + "/commons/beanutils/source/commons-beanutils-1.8.0-BETA-src.tar.gz": "1.8.0-BETA", + "/cocoon/BINARIES/cocoon-2.0.3-vm14-bin.tar.gz": "2.0.3-vm14", + "/felix/xliff_filters_v1_2_7_unix.jar": "v1_2_7", + "/excalibur/releases/200702/excalibur-javadoc-r508111-15022007.tar.gz": "r508111-15022007", + "/geronimo/eclipse/updates/features/org.apache.geronimo.v20.feature_2.0.0.jar": "v20.feature_2.0.0", + "/geronimo/2.1.6/axis2-jaxws-1.3-G20090406.jar": "1.3-G20090406", + "/cassandra/debian/pool/main/c/cassandra/cassandra_0.4.0~beta1-1.diff.gz": "0.4.0~beta1-1", + "/ha-api-3.1.6.jar": "3.1.6", + "ha-api-3.1.6.jar": "3.1.6", } # FIXME: generate a test function for each case for path in data: expected = data[path] - if not expected.lower().startswith('v'): - expected = 'v ' + expected + if not expected.lower().startswith("v"): + expected = "v " + expected assert version.hint(path) == expected From a4e741b413be921e9904b4ece9461478294e4a22 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Thu, 5 Sep 2024 11:59:45 +0200 Subject: [PATCH 021/120] Update CHANGELOG Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d1c440a5..2262567f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,9 +4,12 @@ Release notes Version 32.0.0 - (2024-09-05) ----------------------------- -- Add new optional argument to generate test data files from a template +- Add new optional argument to generate YAML test data files from a template - Migrate URLs to new aboutcode.org org - Drop support for Python 3.7 +- Replace charset_normalizer by chardet because of unstable behavior between minor versions + See https://github.com/jawah/charset_normalizer/issues/520 +- Adopt black and isort style Version 31.2.1 - (2024-05-16) From b6ace696e9b27f42c443b44a327dffb2764a498b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 6 Dec 2024 18:00:48 +0100 Subject: [PATCH 022/120] Compute checksums from binary stream Scanning a large file may exhaust memory and file because commoncode.hash loads the whole file in memory. With this commit, the files are loaded in chunks so we do not run out of memory on very large files, such as VM images. Reference: https://github.com/aboutcode-org/commoncode/issues/72 Signed-off-by: Philippe Ombredanne --- src/commoncode/hash.py | 225 ++++++++++++++++++++++++++++++----------- tests/test_hash.py | 18 ++++ 2 files changed, 186 insertions(+), 57 deletions(-) diff --git a/src/commoncode/hash.py b/src/commoncode/hash.py index 954d3f4b..78d13d7d 100644 --- a/src/commoncode/hash.py +++ b/src/commoncode/hash.py @@ -8,7 +8,9 @@ import binascii import hashlib +import os import sys + from functools import partial from commoncode import filetype @@ -26,33 +28,74 @@ """ -def _hash_mod(bitsize, hmodule): +class Hashable: """ - Return a hashing class returning hashes with a `bitsize` bit length. The - interface of this class is similar to the hash module API. + A mixin for hashers that provides the base methods. """ + # digest_size = length of binary digest for this hash + # binh = binary hasher module + # msg_len = length in bytes of the messages hashed + # total_length = total length in bytes of the messages hashed - class hasher(object): - def __init__(self, msg=None): - self.digest_size = bitsize // 8 - self.h = msg and hmodule(msg).digest()[: self.digest_size] or None + def digest(self): + """ + Return a bytes string digest for this hash. + """ + if not self.msg_len: + return + return self.binh.digest()[: self.digest_size] - def digest(self): - return bytes(self.h) + def hexdigest(self): + """ + Return a string hex digest for this hash. + """ + return self.msg_len and binascii.hexlify(self.digest()).decode("utf-8") - def hexdigest(self): - return self.h and binascii.hexlify(self.h).decode("utf-8") + def b64digest(self): + """ + Return a string base64 digest for this hash. + """ + return self.msg_len and urlsafe_b64encode(self.digest()).decode("utf-8") + + def intdigest(self): + """ + Return a int digest for this hash. + """ + return self.msg_len and int(bin_to_num(self.digest())) + + +def _hash_mod(bitsize, hmodule): + """ + Return a hasher class that returns hashes with a ``bitsize`` bit length. The interface of this + class is similar to the hash module API. + """ - def b64digest(self): - return self.h and urlsafe_b64encode(self.h).decode("utf-8") + class hasher(Hashable): + """A hasher class that behaves like a hashlib module.""" - def intdigest(self): - return self.h and int(bin_to_num(self.h)) + def __init__(self, msg=None, **kwargs): + """ + Return a hasher, populated with an initial ``msg`` bytes string. + Close on the bitsize and hmodule + """ + self.digest_size = bitsize // 8 + self.binh = hmodule() + self.msg_len = 0 + if msg: + self.update(msg) + + def update(self, msg=None): + """ + Update this hash with a ``msg`` bytes string. + """ + if msg: + self.binh.update(msg) + self.msg_len += len(msg) return hasher -# for FIPS support +# for FIPS support, we declare that "usedforsecurity" is False sys_v0 = sys.version_info[0] sys_v1 = sys.version_info[1] if sys_v0 == 3 and sys_v1 >= 9: @@ -60,7 +103,6 @@ def intdigest(self): else: md5_hasher = hashlib.md5 - # Base hashers for each bit size _hashmodules_by_bitsize = { # md5-based @@ -82,31 +124,62 @@ def get_hasher(bitsize): return _hashmodules_by_bitsize[bitsize] -class sha1_git_hasher(object): +class sha1_git_hasher(Hashable): """ Hash content using the git blob SHA1 convention. + See https://git-scm.com/book/en/v2/Git-Internals-Git-Objects#_object_storage """ - def __init__(self, msg=None): + def __init__(self, msg=None, total_length=0, **kwargs): + """ + Initialize a sha1_git_hasher with an optional ``msg`` byte string. The ``total_length`` of + all content that will be hashed, combining the ``msg`` length plus any later call to + update() with additional messages. + """ self.digest_size = 160 // 8 - self.h = msg and self._compute(msg) or None + self.msg_len = 0 - def _compute(self, msg): - # note: bytes interpolation is new in Python 3.5 - git_blob_msg = b"blob %d\0%s" % (len(msg), msg) - return hashlib.sha1(git_blob_msg).digest() + if msg: + self.msg_len = msg_len = len(msg) - def digest(self): - return bytes(self.h) + if not total_length: + total_length = msg_len + else: + if total_length < msg_len: + raise ValueError( + f"Initial msg length: {msg_len} " + f"cannot be larger than the the total_length: {self.total_length}" + ) - def hexdigest(self): - return self.h and binascii.hexlify(self.h).decode("utf-8") + if not total_length: + raise ValueError("total_length cannot be zero") - def b64digest(self): - return self.h and urlsafe_b64encode(self.h).decode("utf-8") + self.total_length = total_length + self.binh = get_hasher(bitsize=160)(total_length=total_length) - def intdigest(self): - return self.h and int(bin_to_num(self.h)) + self._hash_header() + if msg: + self.update(msg) + + def _hash_header(self): + # note: bytes interpolation is new in Python 3.5 + git_blob_header = b"blob %d\0" % (self.total_length) + self.binh.update(msg=git_blob_header) + + def update(self, msg=None): + """ + Update this hash with a ``msg`` bytes string. + """ + if msg: + msg_len = len(msg) + if (msg_len + self.msg_len) > self.total_length: + raise ValueError( + f"Actual combined msg lengths: initial: {self.msg_len} plus added: {msg_len} " + f"cannot be larger than the the total_length: {self.total_length}" + ) + + self.binh.update(msg) + self.msg_len += msg_len _hashmodules_by_name = { @@ -119,25 +192,47 @@ def intdigest(self): } +def get_hasher_instance_by_name(name, total_length=0): + """ + Return a hasher instance for a checksum algorithm ``name`` with a planned ``total_length`` of + bytes to hash. + """ + try: + hm = _hashmodules_by_name[name] + return hm(total_length=total_length) + except KeyError: + raise ValueError(f"Unknown checksum algorithm: {name!r}") + + +def get_file_size(location): + return os.path.getsize(location) + + def checksum(location, name, base64=False): """ - Return a checksum of `bitsize` length from the content of the file at - `location`. The checksum is a hexdigest or base64-encoded is `base64` is - True. + Return a checksum from the content of the file at ``location`` using the ``name`` checksum + algorithm. The checksum is a string as a hexdigest or is base64-encoded is ``base64`` is True. """ if not filetype.is_file(location): return - hasher = _hashmodules_by_name[name] - # fixme: we should read in chunks? - with open(location, "rb") as f: - hashable = f.read() + total_length = get_file_size(location) + chunks = binary_chunks(location) + return checksum_from_chunks(chunks=chunks, total_length=total_length, name=name, base64=base64) - hashed = hasher(hashable) - if base64: - return hashed.b64digest() - return hashed.hexdigest() +def checksum_from_chunks(chunks, name, total_length=0, base64=False): + """ + Return a checksum from the content of the iterator of byte strings ``chunks`` with a + ``total_length`` combined length using the ``name`` checksum algorithm. The returned checksum is + a string as a hexdigest or is base64-encoded is ``base64`` is True. + """ + hasher = get_hasher_instance_by_name(name=name, total_length=total_length) + for chunk in chunks: + hasher.update(chunk) + if base64: + return hasher.b64digest() + return hasher.hexdigest() def md5(location): @@ -164,21 +259,37 @@ def sha1_git(location): return checksum(location, name="sha1_git", base64=False) -def multi_checksums(location, checksum_names=("md5", "sha1", "sha256", "sha512", "sha1_git")): +def binary_chunks(location, size=2 ** 24): """ - Return a mapping of hexdigest checksums keyed by checksum name from the content - of the file at `location`. Use the `checksum_names` list of checksum names. - The mapping is guaranted to contains all the requested names as keys. - If the location is not a file, the values are None. + Read file at ``location`` as binary and yield bytes of up to ``size`` length in bytes, + defaulting to 2**24 bytes, e.g., about 16 MB. """ - results = dict([(name, None) for name in checksum_names]) - if not filetype.is_file(location): - return results - - # fixme: we should read in chunks? with open(location, "rb") as f: - hashable = f.read() + while True: + chunk = f.read(size) + if not chunk: + break + yield chunk - for name in checksum_names: - results[name] = _hashmodules_by_name[name](hashable).hexdigest() - return results + +def multi_checksums(location, checksum_names=("md5", "sha1", "sha256", "sha512", "sha1_git")): + """ + Return a mapping of hexdigest checksum strings keyed by checksum algorithm name from hashing the + content of the file at ``location``. Use the ``checksum_names`` list of checksum names. The + mapping is guaranted to contains all the requested names as keys. If the location is not a file, + or if the file is empty, the values are None. + The purpose of this function is + """ + if not filetype.is_file(location): + return {name: None for name in checksum_names} + file_size = get_file_size(location) + hashers = { + name: get_hasher_instance_by_name(name=name, total_length=file_size) + for name in checksum_names + } + + for chunk in binary_chunks(location): + for hasher in hashers.values(): + hasher.update(msg=chunk) + + return {name: hasher.hexdigest() for name, hasher in hashers.items()} diff --git a/tests/test_hash.py b/tests/test_hash.py index 27d28653..e0640742 100644 --- a/tests/test_hash.py +++ b/tests/test_hash.py @@ -7,9 +7,11 @@ # import os +import hashlib from commoncode.hash import b64sha1 from commoncode.hash import checksum +from commoncode.hash import checksum_from_chunks from commoncode.hash import get_hasher from commoncode.hash import md5 from commoncode.hash import multi_checksums @@ -174,3 +176,19 @@ def test_sha1_git_checksum(self): test_file = self.get_test_loc(test_file) # test that we match the git hash-object assert sha1_git(test_file) == expected_sha1_git + + def test_checksum_from_chunks_can_stream_gigabytes(self): + chunk_16mb = b"0" * 16000000 + chunks_3dot2gb = (chunk_16mb for _ in range(200)) + result = checksum_from_chunks(chunks=chunks_3dot2gb, total_length=16000000 * 200, name="sha1_git") + assert result == "494caf26c43c4473f6e930b0f5c2ecf8121bcf24" + + def test_checksum_from_chunks_from_stream_is_same_as_plain(self): + chunk = b"0" * 16000 + chunks = (chunk for _ in range(100)) + result1 = checksum_from_chunks(chunks=chunks, name="sha256") + + result2 = hashlib.sha256() + for _ in range(100): + result2.update(chunk) + assert result1 == result2.hexdigest() From a0e4c7fede00f78d52763eea3e6726d2a8c146a5 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 6 Dec 2024 18:06:49 +0100 Subject: [PATCH 023/120] Apply formatting Signed-off-by: Philippe Ombredanne --- src/commoncode/hash.py | 4 ++-- tests/test_hash.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/commoncode/hash.py b/src/commoncode/hash.py index 78d13d7d..af992753 100644 --- a/src/commoncode/hash.py +++ b/src/commoncode/hash.py @@ -10,7 +10,6 @@ import hashlib import os import sys - from functools import partial from commoncode import filetype @@ -32,6 +31,7 @@ class Hashable: """ A mixin for hashers that provides the base methods. """ + # digest_size = length of binary digest for this hash # binh = binary hasher module # msg_len = length in bytes of the messages hashed @@ -259,7 +259,7 @@ def sha1_git(location): return checksum(location, name="sha1_git", base64=False) -def binary_chunks(location, size=2 ** 24): +def binary_chunks(location, size=2**24): """ Read file at ``location`` as binary and yield bytes of up to ``size`` length in bytes, defaulting to 2**24 bytes, e.g., about 16 MB. diff --git a/tests/test_hash.py b/tests/test_hash.py index e0640742..52e36bfa 100644 --- a/tests/test_hash.py +++ b/tests/test_hash.py @@ -6,8 +6,8 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -import os import hashlib +import os from commoncode.hash import b64sha1 from commoncode.hash import checksum @@ -180,7 +180,9 @@ def test_sha1_git_checksum(self): def test_checksum_from_chunks_can_stream_gigabytes(self): chunk_16mb = b"0" * 16000000 chunks_3dot2gb = (chunk_16mb for _ in range(200)) - result = checksum_from_chunks(chunks=chunks_3dot2gb, total_length=16000000 * 200, name="sha1_git") + result = checksum_from_chunks( + chunks=chunks_3dot2gb, total_length=16000000 * 200, name="sha1_git" + ) assert result == "494caf26c43c4473f6e930b0f5c2ecf8121bcf24" def test_checksum_from_chunks_from_stream_is_same_as_plain(self): From d04df2cdff84e52bc9b2d11e457944a8981a27c9 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Fri, 6 Dec 2024 18:08:59 +0100 Subject: [PATCH 024/120] Update CHANGELOG Signed-off-by: Philippe Ombredanne --- CHANGELOG.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2262567f..d2e581be 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,13 @@ Release notes ============= + +Version 32.1.0 - (2024-12-06) +----------------------------- + +- Compute file checksums from streaming the file content in chunks to avoid running out of memory + + Version 32.0.0 - (2024-09-05) ----------------------------- From ef2cfb15bc21d5d01d38ac3e73a3257dd589c938 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Fri, 6 Dec 2024 23:55:39 +0530 Subject: [PATCH 025/120] Update docstirngs and python version Signed-off-by: Ayan Sinha Mahapatra --- setup.cfg | 2 +- src/commoncode/hash.py | 116 ++++++++++++++++++++++------------------- 2 files changed, 63 insertions(+), 55 deletions(-) diff --git a/setup.cfg b/setup.cfg index 163d21b4..f606ece3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,7 +37,7 @@ zip_safe = false setup_requires = setuptools_scm[toml] >= 4 -python_requires = >=3.8 +python_requires = >=3.9 install_requires = attrs >= 18.1, !=20.1.0 diff --git a/src/commoncode/hash.py b/src/commoncode/hash.py index af992753..772f9c7e 100644 --- a/src/commoncode/hash.py +++ b/src/commoncode/hash.py @@ -26,42 +26,8 @@ Checksums are operating on files. """ - -class Hashable: - """ - A mixin for hashers that provides the base methods. - """ - - # digest_size = length of binary digest for this hash - # binh = binary hasher module - # msg_len = length in bytes of the messages hashed - # total_length = total length in bytes of the messages hashed - - def digest(self): - """ - Return a bytes string digest for this hash. - """ - if not self.msg_len: - return - return self.binh.digest()[: self.digest_size] - - def hexdigest(self): - """ - Return a string hex digest for this hash. - """ - return self.msg_len and binascii.hexlify(self.digest()).decode("utf-8") - - def b64digest(self): - """ - Return a string base64 digest for this hash. - """ - return self.msg_len and urlsafe_b64encode(self.digest()).decode("utf-8") - - def intdigest(self): - """ - Return a int digest for this hash. - """ - return self.msg_len and int(bin_to_num(self.digest())) +# This is ~16 MB +FILE_CHUNK_SIZE = 2**24 def _hash_mod(bitsize, hmodule): @@ -78,9 +44,15 @@ def __init__(self, msg=None, **kwargs): Return a hasher, populated with an initial ``msg`` bytes string. Close on the bitsize and hmodule """ + # length of binary digest for this hash self.digest_size = bitsize // 8 + + # binh = binary hasher module self.binh = hmodule() + + # msg_len = length in bytes of the message hashed self.msg_len = 0 + if msg: self.update(msg) @@ -95,13 +67,43 @@ def update(self, msg=None): return hasher +class Hashable: + """ + A mixin for hashers that provides the base methods. + """ + + def digest(self): + """ + Return a bytes string digest for this hash. + """ + if not self.msg_len: + return + return self.binh.digest()[: self.digest_size] + + def hexdigest(self): + """ + Return a string hex digest for this hash. + """ + return self.msg_len and binascii.hexlify(self.digest()).decode("utf-8") + + def b64digest(self): + """ + Return a string base64 digest for this hash. + """ + return self.msg_len and urlsafe_b64encode(self.digest()).decode("utf-8") + + def intdigest(self): + """ + Return a int digest for this hash. + """ + return self.msg_len and int(bin_to_num(self.digest())) + + # for FIPS support, we declare that "usedforsecurity" is False sys_v0 = sys.version_info[0] sys_v1 = sys.version_info[1] -if sys_v0 == 3 and sys_v1 >= 9: - md5_hasher = partial(hashlib.md5, usedforsecurity=False) -else: - md5_hasher = hashlib.md5 +md5_hasher = partial(hashlib.md5, usedforsecurity=False) + # Base hashers for each bit size _hashmodules_by_bitsize = { @@ -135,6 +137,9 @@ def __init__(self, msg=None, total_length=0, **kwargs): Initialize a sha1_git_hasher with an optional ``msg`` byte string. The ``total_length`` of all content that will be hashed, combining the ``msg`` length plus any later call to update() with additional messages. + + Here ``total_length`` is total length in bytes of all the messages (chunks) hashed + in contrast to ``msg_len`` which is the length in bytes for the optional message. """ self.digest_size = 160 // 8 self.msg_len = 0 @@ -235,6 +240,19 @@ def checksum_from_chunks(chunks, name, total_length=0, base64=False): return hasher.hexdigest() +def binary_chunks(location, size=FILE_CHUNK_SIZE): + """ + Read file at ``location`` as binary and yield bytes of up to ``size`` length in bytes, + defaulting to 2**24 bytes, e.g., about 16 MB. + """ + with open(location, "rb") as f: + while True: + chunk = f.read(size) + if not chunk: + break + yield chunk + + def md5(location): return checksum(location, name="md5", base64=False) @@ -259,26 +277,16 @@ def sha1_git(location): return checksum(location, name="sha1_git", base64=False) -def binary_chunks(location, size=2**24): - """ - Read file at ``location`` as binary and yield bytes of up to ``size`` length in bytes, - defaulting to 2**24 bytes, e.g., about 16 MB. - """ - with open(location, "rb") as f: - while True: - chunk = f.read(size) - if not chunk: - break - yield chunk - - def multi_checksums(location, checksum_names=("md5", "sha1", "sha256", "sha512", "sha1_git")): """ Return a mapping of hexdigest checksum strings keyed by checksum algorithm name from hashing the content of the file at ``location``. Use the ``checksum_names`` list of checksum names. The mapping is guaranted to contains all the requested names as keys. If the location is not a file, or if the file is empty, the values are None. - The purpose of this function is + + The purpose of this function is to return a set of checksums for a supported set of checksum + algorithms for a given location. This is an API function used in ScanCode --info plugin to get + checksum values. """ if not filetype.is_file(location): return {name: None for name in checksum_names} From 359f72bdac7dbd5fb0befb9127b2605d9fabfc84 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Sat, 7 Dec 2024 00:20:13 +0530 Subject: [PATCH 026/120] Add style checks to CI Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/docs-ci.yml | 7 ++++--- Makefile | 14 +++++++++++--- etc/scripts/utils_thirdparty.py | 6 +++--- src/commoncode/cliutils.py | 4 +--- src/commoncode/command.py | 2 +- src/commoncode/fileset.py | 2 +- src/commoncode/fileutils.py | 2 +- src/commoncode/hash.py | 5 ++--- src/commoncode/resource.py | 7 ++++--- src/commoncode/system.py | 13 +++++++------ src/commoncode/text.py | 2 +- src/commoncode/timeutils.py | 2 +- src/commoncode/version.py | 3 ++- 13 files changed, 39 insertions(+), 30 deletions(-) diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 8c2abfe9..929abd7b 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -1,4 +1,4 @@ -name: CI Documentation +name: CI Documentation and Code style on: [push, pull_request] @@ -21,7 +21,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Dependencies - run: pip install -e .[docs] + run: pip install -e .[docs,testing] - name: Check Sphinx Documentation build minimally working-directory: ./docs @@ -31,4 +31,5 @@ jobs: working-directory: ./docs run: ./scripts/doc8_style_check.sh - + - name: Check for Code style errors + run: make check-ci diff --git a/Makefile b/Makefile index 94451b33..98407413 100644 --- a/Makefile +++ b/Makefile @@ -33,11 +33,19 @@ valid: isort black check: @echo "-> Run pycodestyle (PEP8) validation" - @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,migrations,settings.py,.cache . + @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,scripts,tests,migrations,settings.py,.cache . @echo "-> Run isort imports ordering validation" - @${ACTIVATE} isort --sl --check-only -l 100 setup.py src tests . + @${ACTIVATE} isort --sl -l 100 src tests setup.py --check-only @echo "-> Run black validation" - @${ACTIVATE} black --check --check -l 100 src tests setup.py + @${ACTIVATE} black --check -l 100 src tests setup.py + +check-ci: + @echo "-> Run pycodestyle (PEP8) validation" + pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,scripts,tests,migrations,settings.py,.cache . + @echo "-> Run isort imports ordering validation" + isort --sl -l 100 src tests setup.py --check-only + @echo "-> Run black validation" + black --check -l 100 src tests setup.py clean: @echo "-> Clean the Python env" diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 46dc7289..569b6057 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -854,7 +854,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from remote: {lic_url}") - except: + except Exception: try: # try licensedb second lic_url = f"{LICENSEDB_API_URL}/{filename}" @@ -867,7 +867,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from licensedb: {lic_url}") - except: + except Exception: msg = f'No text for license {filename} in expression "{self.license_expression}" from {self}' print(msg) errors.append(msg) @@ -1302,7 +1302,7 @@ def is_pure(self): def is_pure_wheel(filename): try: return Wheel.from_filename(filename).is_pure() - except: + except Exception: return False diff --git a/src/commoncode/cliutils.py b/src/commoncode/cliutils.py index 1638088d..8ec419c4 100644 --- a/src/commoncode/cliutils.py +++ b/src/commoncode/cliutils.py @@ -10,9 +10,7 @@ import sys import click - -# FIXME: this is NOT API -from click._termui_impl import ProgressBar +from click._termui_impl import ProgressBar # FIXME: this is NOT API from click.termui import style from click.types import BoolParamType from click.utils import echo diff --git a/src/commoncode/command.py b/src/commoncode/command.py index 2570a05a..419da26d 100644 --- a/src/commoncode/command.py +++ b/src/commoncode/command.py @@ -203,7 +203,7 @@ def close_pipe(p): # Ensure process death otherwise proc.wait may hang in some cases # NB: this will run only on POSIX OSes supporting signals os.kill(proc.pid, signal.SIGKILL) # NOQA - except: + except Exception: pass # This may slow things down a tad on non-POSIX Oses but is safe: diff --git a/src/commoncode/fileset.py b/src/commoncode/fileset.py index 9ebab26b..f0298ea6 100644 --- a/src/commoncode/fileset.py +++ b/src/commoncode/fileset.py @@ -161,7 +161,7 @@ def load(location): assert os.path.exists(fn) and os.path.isfile(fn), msg mode = "r" with open(fn, mode) as f: - return [l.strip() for l in f if l and l.strip()] + return [line.strip() for line in f if line and line.strip()] def includes_excludes(patterns, message): diff --git a/src/commoncode/fileutils.py b/src/commoncode/fileutils.py index 123a7175..3f6f9ec5 100644 --- a/src/commoncode/fileutils.py +++ b/src/commoncode/fileutils.py @@ -557,7 +557,7 @@ def _rm_handler(function, path, excinfo): # NOQA elif function == os.remove: try: delete(path, _err_handler=None) - except: + except Exception: pass if os.path.exists(path): diff --git a/src/commoncode/hash.py b/src/commoncode/hash.py index 772f9c7e..5e71fde5 100644 --- a/src/commoncode/hash.py +++ b/src/commoncode/hash.py @@ -284,9 +284,8 @@ def multi_checksums(location, checksum_names=("md5", "sha1", "sha256", "sha512", mapping is guaranted to contains all the requested names as keys. If the location is not a file, or if the file is empty, the values are None. - The purpose of this function is to return a set of checksums for a supported set of checksum - algorithms for a given location. This is an API function used in ScanCode --info plugin to get - checksum values. + The purpose of this function is to avoid read the same file multiple times + to compute different checksums. """ if not filetype.is_file(location): return {name: None for name in checksum_names} diff --git a/src/commoncode/resource.py b/src/commoncode/resource.py index 6ce1536a..5a0fa34c 100644 --- a/src/commoncode/resource.py +++ b/src/commoncode/resource.py @@ -511,7 +511,8 @@ def _create_resources_from_paths(self, root, paths): ) if not newpar: raise Exception( - f"ERROR: Codebase._create_resources_from_paths: cannot create parent for: {parent_path!r}" + "ERROR: Codebase._create_resources_from_paths:" + f" cannot create parent for: {parent_path!r}" ) parent = newpar @@ -1686,7 +1687,7 @@ def _get_scan_data_helper(self, location): """ try: return json.loads(location) - except: + except Exception: location = abspath(normpath(expanduser(location))) with open(location) as f: scan_data = json.load(f) @@ -1842,7 +1843,7 @@ def _populate(self, scan_data): ########################################################## for attr_name in self.codebase_attributes: value = scan_data.get(attr_name) - if value == None: + if not value: continue setattr(self.attributes, attr_name, value) diff --git a/src/commoncode/system.py b/src/commoncode/system.py index 0566a6c0..ea6f7d36 100644 --- a/src/commoncode/system.py +++ b/src/commoncode/system.py @@ -113,12 +113,13 @@ def has_case_sensitive_fs(): case sensitive by default, newer macOS use APFS which is no longer case sensitive by default. - From https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/APFS_Guide/FAQ/FAQ.html - How does Apple File System handle filenames? - APFS accepts only valid UTF-8 encoded filenames for creation, and preserves - both case and normalization of the filename on disk in all variants. APFS, - like HFS+, is case-sensitive on iOS and is available in case-sensitive and - case-insensitive variants on macOS, with case-insensitive being the default. + From + https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/APFS_Guide/FAQ/FAQ.html + How does Apple File System handle filenames? + APFS accepts only valid UTF-8 encoded filenames for creation, and preserves + both case and normalization of the filename on disk in all variants. APFS, + like HFS+, is case-sensitive on iOS and is available in case-sensitive and + case-insensitive variants on macOS, with case-insensitive being the default. """ return not os.path.exists(__file__.upper()) diff --git a/src/commoncode/text.py b/src/commoncode/text.py index ab56b0ae..04df60fc 100644 --- a/src/commoncode/text.py +++ b/src/commoncode/text.py @@ -38,7 +38,7 @@ def lines(s): splitlines. """ # FIXME: leverage new Pythin 3.8 scopeing rules - return [l.strip() for l in s.splitlines() if l.strip()] + return [line.strip() for line in s.splitlines() if line.strip()] def foldcase(text): diff --git a/src/commoncode/timeutils.py b/src/commoncode/timeutils.py index bab8dad7..45caf595 100644 --- a/src/commoncode/timeutils.py +++ b/src/commoncode/timeutils.py @@ -83,7 +83,7 @@ def tstamp2time(stamp): # deal with optional microsec try: microsec = dt_ms[1] - except: + except Exception: microsec = None if microsec: microsec = int(microsec) diff --git a/src/commoncode/version.py b/src/commoncode/version.py index a99ac8d6..70746e80 100644 --- a/src/commoncode/version.py +++ b/src/commoncode/version.py @@ -228,7 +228,8 @@ def get_nupkg_nv(filename): """ Return a NameVersion tuple parsed from the .nupkg NuGet archive `filename`. - For example (taken from https://stackoverflow.com/questions/51662737/regex-to-parse-package-name-and-version-number-from-nuget-package-filenames/51662926): + For example (taken from + https://stackoverflow.com/questions/51662737/regex-to-parse-package-name-and-version-number-from-nuget-package-filenames/51662926): >>> get_nupkg_nv('knockoutjs.3.4.2.nupkg') NameVersion(name='knockoutjs', version='3.4.2') >>> get_nupkg_nv('log4net.2.0.8.nupkg') From 395b971d6723294d0120a5be74963472c1375226 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Sat, 7 Dec 2024 00:47:57 +0530 Subject: [PATCH 027/120] Add CHANGELOG entry (#74) Signed-off-by: Ayan Sinha Mahapatra --- CHANGELOG.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d2e581be..c125634b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,7 +6,7 @@ Version 32.1.0 - (2024-12-06) ----------------------------- - Compute file checksums from streaming the file content in chunks to avoid running out of memory - +- Drop support for python 3.8 and add support for python 3.12 Version 32.0.0 - (2024-09-05) ----------------------------- From a92905297acf39ecd820bfb133f8670c39b40c97 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Fri, 17 Jan 2025 20:07:25 +0100 Subject: [PATCH 028/120] Drop deprecated macos-12 runner --- azure-pipelines.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c2a3b522..39601e62 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -23,14 +23,6 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos12_cpython - image_name: macOS-12 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml parameters: job_name: macos13_cpython From 8fbe7f2548fe7263c998da837f87c52fed53d81e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Fri, 17 Jan 2025 09:57:16 +0100 Subject: [PATCH 029/120] migrate os.listdir() to os.scandir() to increase performance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As recommended in https://peps.python.org/pep-0471/ Signed-off-by: Miroslav Suchý --- src/commoncode/fileutils.py | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/src/commoncode/fileutils.py b/src/commoncode/fileutils.py index 3f6f9ec5..651006ae 100644 --- a/src/commoncode/fileutils.py +++ b/src/commoncode/fileutils.py @@ -321,8 +321,6 @@ def walk(location, ignored=None, follow_symlinks=False): If `follow_symlinks` is True, then symlinks will not be ignored and be collected like regular files and directories """ - # TODO: consider using the new "scandir" module for some speed-up. - is_ignored = ignored(location) if ignored else False if is_ignored: if TRACE: @@ -335,15 +333,10 @@ def walk(location, ignored=None, follow_symlinks=False): elif filetype.is_dir(location, follow_symlinks=follow_symlinks): dirs = [] files = [] - # TODO: consider using scandir - for name in os.listdir(location): - loc = os.path.join(location, name) + for entry in os.scandir(location): + loc = os.path.join(location, entry.name) if filetype.is_special(loc) or (ignored and ignored(loc)): - if ( - follow_symlinks - and filetype.is_link(loc) - and not filetype.is_broken_link(location) - ): + if follow_symlinks and entry.is_symlink() and not filetype.is_broken_link(location): pass else: if TRACE: @@ -351,10 +344,10 @@ def walk(location, ignored=None, follow_symlinks=False): logger_debug("walk: ignored:", loc, ign) continue # special files and symlinks are always ignored - if filetype.is_dir(loc, follow_symlinks=follow_symlinks): - dirs.append(name) - elif filetype.is_file(loc, follow_symlinks=follow_symlinks): - files.append(name) + if entry.is_dir(follow_symlinks=follow_symlinks): + dirs.append(entry.name) + elif entry.is_file(follow_symlinks=follow_symlinks): + files.append(entry.name) yield location, dirs, files for dr in dirs: From 4af4fce3cc57d001c6c26f77d477cd44cef2ffef Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Sat, 15 Feb 2025 00:09:49 +0530 Subject: [PATCH 030/120] Update CI/Actions runners Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/docs-ci.yml | 8 ++++---- .github/workflows/pypi-release.yml | 18 +++++++++--------- azure-pipelines.yml | 22 +++++++++++----------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 8c2abfe9..621de4b2 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -4,19 +4,19 @@ on: [push, pull_request] jobs: build: - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: max-parallel: 4 matrix: - python-version: [3.9] + python-version: [3.12] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index d2206c87..a66c9c80 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -21,14 +21,14 @@ on: jobs: build-pypi-distribs: name: Build and publish library to PyPI - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.12 - name: Install pypa/build run: python -m pip install build --user @@ -37,7 +37,7 @@ jobs: run: python -m build --sdist --wheel --outdir dist/ - name: Upload built archives - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: pypi_archives path: dist/* @@ -47,17 +47,17 @@ jobs: name: Create GH release needs: - build-pypi-distribs - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Download built archives - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: pypi_archives path: dist - name: Create GH release - uses: softprops/action-gh-release@v1 + uses: softprops/action-gh-release@v2 with: draft: true files: dist/* @@ -67,11 +67,11 @@ jobs: name: Create PyPI release needs: - create-gh-release - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 steps: - name: Download built archives - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: pypi_archives path: dist diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 39601e62..a220f2be 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -9,17 +9,17 @@ jobs: - template: etc/ci/azure-posix.yml parameters: - job_name: ubuntu20_cpython - image_name: ubuntu-20.04 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + job_name: ubuntu22_cpython + image_name: ubuntu-22.04 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs - template: etc/ci/azure-posix.yml parameters: - job_name: ubuntu22_cpython - image_name: ubuntu-22.04 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + job_name: ubuntu24_cpython + image_name: ubuntu-24.04 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -27,7 +27,7 @@ jobs: parameters: job_name: macos13_cpython image_name: macOS-13 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -35,7 +35,7 @@ jobs: parameters: job_name: macos14_cpython_arm64 image_name: macOS-14 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +43,7 @@ jobs: parameters: job_name: macos14_cpython image_name: macOS-14-large - python_versions: ['3.8', '3.8', '3.9', '3.10', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -51,7 +51,7 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -59,6 +59,6 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From 8bf864f109900622d23c76aad4b0f19d1a48db7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20L=C3=B6hel?= Date: Fri, 14 Feb 2025 13:17:58 -0600 Subject: [PATCH 031/120] Adjust `as_unicode` for breaking bs4 4.13 changes (#80) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adjust `as_unicode` for breaking bs4 4.13 changes Fixes: #79 Reference: https://git.launchpad.net/beautifulsoup/tree/CHANGELOG#n431 Signed-off-by: Jürgen Löhel <2422012+jloehel@users.noreply.github.com> --- requirements.txt | 2 +- setup.cfg | 2 +- src/commoncode/text.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index e4b25aa3..562ae0ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ attrs==22.2.0 -beautifulsoup4==4.11.2 +beautifulsoup4==4.13.3 certifi==2022.12.7 chardet==5.2.0 click==8.1.3 diff --git a/setup.cfg b/setup.cfg index 353b78f5..063b149e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -41,7 +41,7 @@ python_requires = >=3.9 install_requires = attrs >= 18.1, !=20.1.0 - Beautifulsoup4[chardet] >= 4.0.0 + Beautifulsoup4[chardet] >= 4.13.0 click >= 6.7, !=7.0 requests[use_chardet_on_py3] >= 2.7.0 saneyaml >= 0.5.2 diff --git a/src/commoncode/text.py b/src/commoncode/text.py index 04df60fc..20c11793 100644 --- a/src/commoncode/text.py +++ b/src/commoncode/text.py @@ -143,4 +143,4 @@ def as_unicode(s): if not s: return s assert isinstance(s, bytes), "s must be bytes but is: {}".format(s) - return UnicodeDammit(s).markup + return UnicodeDammit(s).unicode_markup From e267b2bff6de9ab330895b687b92eb58a83c3420 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Sat, 15 Feb 2025 01:06:01 +0530 Subject: [PATCH 032/120] Update CHANGELOG for release v32.2.0 Signed-off-by: Ayan Sinha Mahapatra --- CHANGELOG.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c125634b..1a583f6e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,13 @@ Release notes ============= +Version 32.2.0 - (2025-02-15) +----------------------------- + +- Adjust as_unicode for breaking bs4 4.13 changes + See https://github.com/aboutcode-org/commoncode/issues/79 +- Add support for python 3.13 +- Drop deprecated CI runners and update CI runners to latest Version 32.1.0 - (2024-12-06) ----------------------------- From 6bc57ab6d9038465e47a6f528899406a21382ae4 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 6 Mar 2025 20:25:24 +0700 Subject: [PATCH 033/120] Fix empty file checksum Reference: https://github.com/aboutcode-org/scancode-toolkit/pull/4149/files#r1983173751 Signed-off-by: Ayan Sinha Mahapatra --- src/commoncode/hash.py | 5 +++++ tests/data/hash/empty | 0 tests/test_hash.py | 5 +++++ 3 files changed, 10 insertions(+) create mode 100644 tests/data/hash/empty diff --git a/src/commoncode/hash.py b/src/commoncode/hash.py index 5e71fde5..b97dd898 100644 --- a/src/commoncode/hash.py +++ b/src/commoncode/hash.py @@ -217,6 +217,8 @@ def checksum(location, name, base64=False): """ Return a checksum from the content of the file at ``location`` using the ``name`` checksum algorithm. The checksum is a string as a hexdigest or is base64-encoded is ``base64`` is True. + + Return None if ``location`` is not a file or an empty file. """ if not filetype.is_file(location): return @@ -290,6 +292,9 @@ def multi_checksums(location, checksum_names=("md5", "sha1", "sha256", "sha512", if not filetype.is_file(location): return {name: None for name in checksum_names} file_size = get_file_size(location) + if file_size == 0: + return {name: None for name in checksum_names} + hashers = { name: get_hasher_instance_by_name(name=name, total_length=file_size) for name in checksum_names diff --git a/tests/data/hash/empty b/tests/data/hash/empty new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_hash.py b/tests/test_hash.py index 52e36bfa..b1ab88e5 100644 --- a/tests/test_hash.py +++ b/tests/test_hash.py @@ -194,3 +194,8 @@ def test_checksum_from_chunks_from_stream_is_same_as_plain(self): for _ in range(100): result2.update(chunk) assert result1 == result2.hexdigest() + + def test_checksum_empty_file(self): + test_file = self.get_test_loc("hash/empty") + checksums = multi_checksums(location=test_file, checksum_names=("sha1",)) + assert checksums == {"sha1": None} From 1963f81735f4724aed0a6d2711822d773b36d97e Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 6 Mar 2025 20:38:56 +0700 Subject: [PATCH 034/120] Migrate remaining os.listdir to os.scandir Signed-off-by: Ayan Sinha Mahapatra --- src/commoncode/filetype.py | 2 +- src/commoncode/fileutils.py | 22 +++++++++++++--------- src/commoncode/resource.py | 2 +- tests/test_fileutils.py | 2 +- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/commoncode/filetype.py b/src/commoncode/filetype.py index a7681d4f..ad1d5561 100644 --- a/src/commoncode/filetype.py +++ b/src/commoncode/filetype.py @@ -233,7 +233,7 @@ def counter(location, counting_function): return count_fun(location) elif is_dir(location): count += sum( - counter(os.path.join(location, p), counting_function) for p in os.listdir(location) + counter(os.path.join(location, p.name), counting_function) for p in os.scandir(location) ) return count diff --git a/src/commoncode/fileutils.py b/src/commoncode/fileutils.py index 651006ae..8d2a1e52 100644 --- a/src/commoncode/fileutils.py +++ b/src/commoncode/fileutils.py @@ -333,10 +333,14 @@ def walk(location, ignored=None, follow_symlinks=False): elif filetype.is_dir(location, follow_symlinks=follow_symlinks): dirs = [] files = [] - for entry in os.scandir(location): - loc = os.path.join(location, entry.name) + for resource in os.scandir(location): + loc = os.path.join(location, resource.name) if filetype.is_special(loc) or (ignored and ignored(loc)): - if follow_symlinks and entry.is_symlink() and not filetype.is_broken_link(location): + if ( + follow_symlinks + and resource.is_symlink() + and not filetype.is_broken_link(location) + ): pass else: if TRACE: @@ -344,10 +348,10 @@ def walk(location, ignored=None, follow_symlinks=False): logger_debug("walk: ignored:", loc, ign) continue # special files and symlinks are always ignored - if entry.is_dir(follow_symlinks=follow_symlinks): - dirs.append(entry.name) - elif entry.is_file(follow_symlinks=follow_symlinks): - files.append(entry.name) + if resource.is_dir(follow_symlinks=follow_symlinks): + dirs.append(resource.name) + elif resource.is_file(follow_symlinks=follow_symlinks): + files.append(resource.name) yield location, dirs, files for dr in dirs: @@ -396,7 +400,7 @@ def copytree(src, dst): if not filetype.is_readable(src): chmod(src, R, recurse=False) - names = os.listdir(src) + names = [resource.name for resource in os.scandir(src)] if not os.path.exists(dst): os.makedirs(dst) @@ -537,7 +541,7 @@ def _rm_handler(function, path, excinfo): # NOQA """ if TRACE: logger_debug("_rm_handler:", "path:", path, "excinfo:", excinfo) - if function in (os.rmdir, os.listdir): + if function in (os.rmdir, os.listdir, os.scandir): try: chmod(path, RW, recurse=True) shutil.rmtree(path, True) diff --git a/src/commoncode/resource.py b/src/commoncode/resource.py index 5a0fa34c..d66476fb 100644 --- a/src/commoncode/resource.py +++ b/src/commoncode/resource.py @@ -292,7 +292,7 @@ def __init__( self.is_file = filetype_is_file(location) # True if this codebase root is a file or an empty directory. - self.has_single_resource = bool(self.is_file or not os.listdir(location)) + self.has_single_resource = bool(self.is_file or not os.scandir(location)) ######################################################################## # Set up caching, summary, timing, and error info diff --git a/tests/test_fileutils.py b/tests/test_fileutils.py index 73cedc1e..6627c216 100644 --- a/tests/test_fileutils.py +++ b/tests/test_fileutils.py @@ -168,7 +168,7 @@ def test_copyfile_does_not_keep_permissions(self): assert not filetype.is_readable(src_file) fileutils.copyfile(src_file, dest) - dest_file = join(dest, os.listdir(dest)[0]) + dest_file = join(dest, list(os.scandir(dest))[0].name) assert filetype.is_readable(dest_file) finally: fileutils.chmod(src_file, fileutils.RW, recurse=True) From aba7c1bdce1477b6b44c4afabd747bd81d9654cf Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 6 Mar 2025 20:50:36 +0700 Subject: [PATCH 035/120] Update CHANGELOG for v32.2.1 Signed-off-by: Ayan Sinha Mahapatra --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1a583f6e..64d94ee1 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Release notes ============= +Version 32.2.1 - (2025-03-06) +----------------------------- + +- migrate ``os.listdir()`` to ``os.scandir()`` to increase performance +- Fix checksums for empty files + Version 32.2.0 - (2025-02-15) ----------------------------- From 320ec21daa249ceae0c07787f9e52134b3ad06ab Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 27 Mar 2025 14:54:31 -0700 Subject: [PATCH 036/120] Replace black and isort with ruff * Use ruff config and Make commands from scancode.io Signed-off-by: Jono Yang --- Makefile | 27 ++++++++++++--------------- pyproject.toml | 37 +++++++++++++++++++++++++++++++++++++ setup.cfg | 3 +-- 3 files changed, 50 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 94451b33..1738b20a 100644 --- a/Makefile +++ b/Makefile @@ -17,27 +17,24 @@ dev: @echo "-> Configure the development envt." ./configure --dev -isort: - @echo "-> Apply isort changes to ensure proper imports ordering" - ${VENV}/bin/isort --sl -l 100 src tests setup.py - -black: - @echo "-> Apply black code formatter" - ${VENV}/bin/black -l 100 src tests setup.py - doc8: @echo "-> Run doc8 validation" @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ -valid: isort black +valid: + @echo "-> Run Ruff format" + @${ACTIVATE} ruff format + @echo "-> Run Ruff linter" + @${ACTIVATE} ruff check --fix check: - @echo "-> Run pycodestyle (PEP8) validation" - @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,migrations,settings.py,.cache . - @echo "-> Run isort imports ordering validation" - @${ACTIVATE} isort --sl --check-only -l 100 setup.py src tests . - @echo "-> Run black validation" - @${ACTIVATE} black --check --check -l 100 src tests setup.py + @echo "-> Run Ruff linter validation (pycodestyle, bandit, isort, and more)" + @${ACTIVATE} ruff check + @echo "-> Run Ruff format validation" + @${ACTIVATE} ruff format --check + @$(MAKE) doc8 + @echo "-> Run ABOUT files validation" + @${ACTIVATE} about check etc/ clean: @echo "-> Clean the Python env" diff --git a/pyproject.toml b/pyproject.toml index cde79074..01e60fc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,3 +50,40 @@ addopts = [ "--strict-markers", "--doctest-modules" ] + +[tool.ruff] +line-length = 88 +extend-exclude = [] +target-version = "py310" + +[tool.ruff.lint] +# Rules: https://docs.astral.sh/ruff/rules/ +select = [ + "E", # pycodestyle + "W", # pycodestyle warnings + "D", # pydocstyle + "F", # Pyflakes + "UP", # pyupgrade + "S", # flake8-bandit + "I", # isort + "C9", # McCabe complexity +] +ignore = ["D1", "D203", "D205", "D212", "D400", "D415"] + +[tool.ruff.lint.isort] +force-single-line = true +sections = { django = ["django"] } +section-order = [ + "future", + "standard-library", + "django", + "third-party", + "first-party", + "local-folder", +] + +[tool.ruff.lint.mccabe] +max-complexity = 10 + +[tool.ruff.lint.per-file-ignores] +# Place paths of files to be ignored by ruff here diff --git a/setup.cfg b/setup.cfg index ef7d369b..aaec643f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,8 +54,7 @@ testing = aboutcode-toolkit >= 7.0.2 pycodestyle >= 2.8.0 twine - black - isort + ruff docs = Sphinx>=5.0.2 From d4e29c36c21ab81797604911cdeaea83d80e8088 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 00:46:06 +0100 Subject: [PATCH 037/120] Use org standard 100 line length Signed-off-by: Philippe Ombredanne --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 01e60fc6..cea91bd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ addopts = [ ] [tool.ruff] -line-length = 88 +line-length = 100 extend-exclude = [] target-version = "py310" From 6c028f7219ae876ea62074ae435e574525e205d6 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 08:40:28 +0100 Subject: [PATCH 038/120] Lint all common code directories Signed-off-by: Philippe Ombredanne --- pyproject.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index cea91bd1..9e627366 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,14 @@ addopts = [ line-length = 100 extend-exclude = [] target-version = "py310" +include = [ + "pyproject.toml", + "src/**/*.py", + "etc/**/*.py", + "test/**/*.py", + "doc/**/*", + "*.py" +] [tool.ruff.lint] # Rules: https://docs.astral.sh/ruff/rules/ From 233f3edabfbab390029fb9f1842bf43766b04583 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 09:07:47 +0100 Subject: [PATCH 039/120] Remove unused targets Signed-off-by: Philippe Ombredanne --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1738b20a..930e8010 100644 --- a/Makefile +++ b/Makefile @@ -48,4 +48,4 @@ docs: rm -rf docs/_build/ @${ACTIVATE} sphinx-build docs/ docs/_build/ -.PHONY: conf dev check valid black isort clean test docs +.PHONY: conf dev check valid clean test docs From 55545bf7a1a8f119a560c7f548ce5a460f39f37d Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 11:03:05 +0100 Subject: [PATCH 040/120] Improve import sorting Signed-off-by: Philippe Ombredanne --- etc/scripts/check_thirdparty.py | 1 - etc/scripts/fetch_thirdparty.py | 2 +- etc/scripts/test_utils_pip_compatibility_tags.py | 3 +-- etc/scripts/utils_dejacode.py | 1 - etc/scripts/utils_pip_compatibility_tags.py | 14 ++++++-------- etc/scripts/utils_thirdparty.py | 3 +-- pyproject.toml | 7 ++++++- 7 files changed, 15 insertions(+), 16 deletions(-) diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 2daded94..62dbb14f 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -12,7 +12,6 @@ import utils_thirdparty - @click.command() @click.option( "-d", diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 3f9ff527..30d376c4 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -16,8 +16,8 @@ import click -import utils_thirdparty import utils_requirements +import utils_thirdparty TRACE = False TRACE_DEEP = False diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py index 98187c56..a33b8b38 100644 --- a/etc/scripts/test_utils_pip_compatibility_tags.py +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -25,14 +25,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -from unittest.mock import patch import sysconfig +from unittest.mock import patch import pytest import utils_pip_compatibility_tags - @pytest.mark.parametrize( "version_info, expected", [ diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index 652252d4..c71543f6 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -14,7 +14,6 @@ import requests import saneyaml - from packvers import version as packaging_version """ diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index af42a0cd..de0ac95d 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -27,14 +27,12 @@ import re -from packvers.tags import ( - compatible_tags, - cpython_tags, - generic_tags, - interpreter_name, - interpreter_version, - mac_platforms, -) +from packvers.tags import compatible_tags +from packvers.tags import cpython_tags +from packvers.tags import generic_tags +from packvers.tags import interpreter_name +from packvers.tags import interpreter_version +from packvers.tags import mac_platforms _osx_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 46dc7289..b0295eca 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -25,14 +25,13 @@ import packageurl import requests import saneyaml +import utils_pip_compatibility_tags from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name from packvers import tags as packaging_tags from packvers import version as packaging_version -import utils_pip_compatibility_tags - """ Utilities to manage Python thirparty libraries source, binaries and metadata in local directories and remote repositories. diff --git a/pyproject.toml b/pyproject.toml index 9e627366..ba55770f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,10 +76,15 @@ select = [ "I", # isort "C9", # McCabe complexity ] -ignore = ["D1", "D203", "D205", "D212", "D400", "D415"] +ignore = ["D1", "D200", "D203", "D205", "D212", "D400", "D415"] [tool.ruff.lint.isort] force-single-line = true +lines-after-imports = 1 +default-section = "first-party" +known-first-party = ["src", "tests", "etc/scripts/**/*.py"] +known-third-party = ["click", "pytest"] + sections = { django = ["django"] } section-order = [ "future", From 0b63e5073b6b1cdc0960abe35060ad0fdb67b665 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 21:35:16 +0100 Subject: [PATCH 041/120] Apply small code updates Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_requirements.py | 20 ++++++++----- etc/scripts/utils_thirdparty.py | 48 +++++++++++++++---------------- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 1c502390..a9ac2235 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -57,21 +57,25 @@ def get_required_name_version(requirement, with_unpinned=False): >>> assert get_required_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") >>> assert get_required_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") >>> assert get_required_name_version("foo", with_unpinned=True) == ("foo", "") - >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_required_name_version("foo>=1.2") + >>> expected = ("foo", ""), get_required_name_version("foo>=1.2") + >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == expected >>> try: ... assert not get_required_name_version("foo", with_unpinned=False) ... except Exception as e: ... assert "Requirement version must be pinned" in str(e) """ requirement = requirement and "".join(requirement.lower().split()) - assert requirement, f"specifier is required is empty:{requirement!r}" + if not requirement: + raise ValueError(f"specifier is required is empty:{requirement!r}") name, operator, version = split_req(requirement) - assert name, f"Name is required: {requirement}" + if not name: + raise ValueError(f"Name is required: {requirement}") is_pinned = operator == "==" if with_unpinned: version = "" else: - assert is_pinned and version, f"Requirement version must be pinned: {requirement}" + if not is_pinned and version: + raise ValueError(f"Requirement version must be pinned: {requirement}") return name, version @@ -120,7 +124,7 @@ def get_installed_reqs(site_packages_dir): # setuptools, pip args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] - return subprocess.check_output(args, encoding="utf-8") + return subprocess.check_output(args, encoding="utf-8") # noqa: S603 comparators = ( @@ -150,9 +154,11 @@ def split_req(req): >>> assert split_req("foo >= 1.2.3 ") == ("foo", ">=", "1.2.3"), split_req("foo >= 1.2.3 ") >>> assert split_req("foo>=1.2") == ("foo", ">=", "1.2"), split_req("foo>=1.2") """ - assert req + if not req: + raise ValueError("req is required") # do not allow multiple constraints and tags - assert not any(c in req for c in ",;") + if not any(c in req for c in ",;"): + raise Exception(f"complex requirements with : or ; not supported: {req}") req = "".join(req.split()) if not any(c in req for c in comparators): return req, "", "" diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index b0295eca..6d5ffdce 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -559,7 +558,8 @@ def download(self, dest_dir=THIRDPARTY_DIR): Download this distribution into `dest_dir` directory. Return the fetched filename. """ - assert self.filename + if not self.filename: + raise ValueError(f"self.filename has no value but is required: {self.filename!r}") if TRACE_DEEP: print( f"Fetching distribution of {self.name}=={self.version}:", @@ -829,10 +829,9 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): urls = LinksRepository.from_url( use_cached_index=use_cached_index).links errors = [] - extra_lic_names = [l.get("file") - for l in self.extra_data.get("licenses", {})] + extra_lic_names = [lic.get("file") for lic in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] - extra_lic_names = [ln for ln in extra_lic_names if ln] + extra_lic_names = [eln for eln in extra_lic_names if eln] lic_names = [f"{key}.LICENSE" for key in self.get_license_keys()] for filename in lic_names + extra_lic_names: floc = os.path.join(dest_dir, filename) @@ -853,7 +852,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from remote: {lic_url}") - except: + except Exception: try: # try licensedb second lic_url = f"{LICENSEDB_API_URL}/{filename}" @@ -866,8 +865,9 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from licensedb: {lic_url}") - except: - msg = f'No text for license {filename} in expression "{self.license_expression}" from {self}' + except Exception: + msg = f"No text for license {filename} in expression " + f"{self.license_expression!r} from {self}" print(msg) errors.append(msg) @@ -1009,7 +1009,7 @@ def get_license_link_for_filename(filename, urls): exception if no link is found or if there are more than one link for that file name. """ - path_or_url = [l for l in urls if l.endswith(f"/{filename}")] + path_or_url = [url for url in urls if url.endswith(f"/{filename}")] if not path_or_url: raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: @@ -1140,7 +1140,6 @@ def to_filename(self): @attr.attributes class Wheel(Distribution): - """ Represents a wheel file. @@ -1301,7 +1300,7 @@ def is_pure(self): def is_pure_wheel(filename): try: return Wheel.from_filename(filename).is_pure() - except: + except Exception: return False @@ -1489,8 +1488,7 @@ def dists_from_paths_or_urls(cls, paths_or_urls): ) except InvalidDistributionFilename: if TRACE_DEEP: - print( - f" Skipping invalid distribution from: {path_or_url}") + print(f" Skipping invalid distribution from: {path_or_url}") continue return dists @@ -1500,8 +1498,7 @@ def get_distributions(self): """ if self.sdist: yield self.sdist - for wheel in self.wheels: - yield wheel + yield from self.wheels def get_url_for_filename(self, filename): """ @@ -1632,7 +1629,8 @@ class PypiSimpleRepository: type=dict, default=attr.Factory(lambda: defaultdict(dict)), metadata=dict( - help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo" + help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} " + "available in this repo" ), ) @@ -1647,7 +1645,8 @@ class PypiSimpleRepository: type=bool, default=False, metadata=dict( - help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache." + help="If True, use any existing on-disk cached PyPI index files. " + "Otherwise, fetch and cache." ), ) @@ -1656,7 +1655,8 @@ def _get_package_versions_map(self, name): Return a mapping of all available PypiPackage version for this package name. The mapping may be empty. It is ordered by version from oldest to newest """ - assert name + if not name: + raise ValueError(f"name is required: {name!r}") normalized_name = NameVer.normalize_name(name) versions = self.packages[normalized_name] if not versions and normalized_name not in self.fetched_package_normalized_names: @@ -1713,7 +1713,7 @@ def fetch_links(self, normalized_name): ) links = collect_urls(text) # TODO: keep sha256 - links = [l.partition("#sha256=") for l in links] + links = [link.partition("#sha256=") for link in links] links = [url for url, _, _sha256 in links] return links @@ -1936,7 +1936,7 @@ def get_remote_file_content( # several redirects and that we can ignore content there. A HEAD request may # not get us this last header print(f" DOWNLOADING: {url}") - with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: + with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113 status = response.status_code if status != requests.codes.ok: # NOQA if status == 429 and _delay < 20: @@ -2161,7 +2161,7 @@ def call(args, verbose=TRACE): """ if TRACE_DEEP: print("Calling:", " ".join(args)) - with subprocess.Popen( + with subprocess.Popen( # noqa: S603 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" ) as process: @@ -2227,7 +2227,7 @@ def download_wheels_with_pip( cli_args.extend(["--requirement", req_file]) if TRACE: - print(f"Downloading wheels using command:", " ".join(cli_args)) + print("Downloading wheels using command:", " ".join(cli_args)) existing = set(os.listdir(dest_dir)) error = False @@ -2260,7 +2260,7 @@ def download_wheels_with_pip( def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603 except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") @@ -2312,5 +2312,5 @@ def get_license_expression(declared_licenses): return get_only_expression_from_extracted_license(declared_licenses) except ImportError: # Scancode is not installed, clean and join all the licenses - lics = [python_safe_name(l).lower() for l in declared_licenses] + lics = [python_safe_name(lic).lower() for lic in declared_licenses] return " AND ".join(lics).lower() From 092f545f5b87442ae22884cb4d5381883343a1c2 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 21:42:03 +0100 Subject: [PATCH 042/120] Format code Signed-off-by: Philippe Ombredanne --- etc/scripts/check_thirdparty.py | 3 +- etc/scripts/fetch_thirdparty.py | 26 ++++----- etc/scripts/gen_pypi_simple.py | 4 +- etc/scripts/utils_dejacode.py | 15 +++--- etc/scripts/utils_requirements.py | 9 ++-- etc/scripts/utils_thirdparty.py | 90 +++++++++++-------------------- 6 files changed, 50 insertions(+), 97 deletions(-) diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 62dbb14f..1aa4e28a 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -16,8 +16,7 @@ @click.option( "-d", "--dest", - type=click.Path(exists=True, readable=True, - path_type=str, file_okay=False), + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, help="Path to the thirdparty directory to check.", ) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 30d376c4..c2246837 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -55,8 +55,7 @@ "-d", "--dest", "dest_dir", - type=click.Path(exists=True, readable=True, - path_type=str, file_okay=False), + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), metavar="DIR", default=utils_thirdparty.THIRDPARTY_DIR, show_default=True, @@ -121,7 +120,7 @@ show_default=False, multiple=True, help="Package name(s) that come only in sdist format (no wheels). " - "The command will not fail and exit if no wheel exists for these names", + "The command will not fail and exit if no wheel exists for these names", ) @click.option( "--wheel-only", @@ -132,7 +131,7 @@ show_default=False, multiple=True, help="Package name(s) that come only in wheel format (no sdist). " - "The command will not fail and exit if no sdist exists for these names", + "The command will not fail and exit if no sdist exists for these names", ) @click.option( "--no-dist", @@ -143,7 +142,7 @@ show_default=False, multiple=True, help="Package name(s) that do not come either in wheel or sdist format. " - "The command will not fail and exit if no distribution exists for these names", + "The command will not fail and exit if no distribution exists for these names", ) @click.help_option("-h", "--help") def fetch_thirdparty( @@ -225,8 +224,7 @@ def fetch_thirdparty( environments = None if wheels: evts = itertools.product(python_versions, operating_systems) - environments = [utils_thirdparty.Environment.from_pyver_and_os( - pyv, os) for pyv, os in evts] + environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] # Collect PyPI repos repos = [] @@ -250,7 +248,6 @@ def fetch_thirdparty( print(f"Processing: {name} @ {version}") if wheels: for environment in environments: - if TRACE: print(f" ==> Fetching wheel for envt: {environment}") @@ -262,14 +259,11 @@ def fetch_thirdparty( repos=repos, ) if not fetched: - wheels_or_sdist_not_found[f"{name}=={version}"].append( - environment) + wheels_or_sdist_not_found[f"{name}=={version}"].append(environment) if TRACE: print(f" NOT FOUND") - if (sdists or - (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only) - ): + if sdists or (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only): if TRACE: print(f" ==> Fetching sdist: {name}=={version}") @@ -292,8 +286,7 @@ def fetch_thirdparty( sdist_missing = sdists and "sdist" in dists and not name in wheel_only if sdist_missing: mia.append(f"SDist missing: {nv} {dists}") - wheels_missing = wheels and any( - d for d in dists if d != "sdist") and not name in sdist_only + wheels_missing = wheels and any(d for d in dists if d != "sdist") and not name in sdist_only if wheels_missing: mia.append(f"Wheels missing: {nv} {dists}") @@ -303,8 +296,7 @@ def fetch_thirdparty( raise Exception(mia) print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") - utils_thirdparty.fetch_abouts_and_licenses( - dest_dir=dest_dir, use_cached_index=use_cached_index) + utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) utils_thirdparty.clean_about_files(dest_dir=dest_dir) # check for problems diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 214d90dc..cfe68e67 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -69,7 +69,6 @@ def get_package_name_from_filename(filename): raise InvalidDistributionFilename(filename) elif filename.endswith(wheel_ext): - wheel_info = get_wheel_from_filename(filename) if not wheel_info: @@ -200,11 +199,10 @@ def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi" simple_html_index = [ "", "PyPI Simple Index", - '' '', + '', ] for pkg_file in directory.iterdir(): - pkg_filename = pkg_file.name if ( diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index c71543f6..cd39cda3 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -32,8 +32,7 @@ def can_do_api_calls(): if not DEJACODE_API_KEY and DEJACODE_API_URL: - print( - "DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") + print("DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") return False else: return True @@ -68,8 +67,7 @@ def get_package_data(distribution): return results[0] elif len_results > 1: - print( - f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") + print(f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") else: print("Could not find package:", distribution.download_url) @@ -150,12 +148,11 @@ def find_latest_dejacode_package(distribution): # there was no exact match, find the latest version # TODO: consider the closest version rather than the latest # or the version that has the best data - with_versions = [(packaging_version.parse(p["version"]), p) - for p in packages] + with_versions = [(packaging_version.parse(p["version"]), p) for p in packages] with_versions = sorted(with_versions) latest_version, latest_package_version = sorted(with_versions)[-1] print( - f"Found DejaCode latest version: {latest_version} " f"for dist: {distribution.package_url}", + f"Found DejaCode latest version: {latest_version} for dist: {distribution.package_url}", ) return latest_package_version @@ -181,7 +178,7 @@ def create_dejacode_package(distribution): } fields_to_carry_over = [ - "download_url" "type", + "download_urltype", "namespace", "name", "version", @@ -209,5 +206,5 @@ def create_dejacode_package(distribution): if response.status_code != 201: raise Exception(f"Error, cannot create package for: {distribution}") - print(f'New Package created at: {new_package_data["absolute_url"]}') + print(f"New Package created at: {new_package_data['absolute_url']}") return new_package_data diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index a9ac2235..167bc9f5 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -106,8 +106,7 @@ def lock_dev_requirements( all_req_nvs = get_required_name_versions(all_req_lines) dev_only_req_nvs = {n: v for n, v in all_req_nvs if n not in main_names} - new_reqs = "\n".join( - f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) + new_reqs = "\n".join(f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) with open(dev_requirements_file, "w") as fo: fo.write(new_reqs) @@ -118,12 +117,10 @@ def get_installed_reqs(site_packages_dir): as a text. """ if not os.path.exists(site_packages_dir): - raise Exception( - f"site_packages directory: {site_packages_dir!r} does not exists") + raise Exception(f"site_packages directory: {site_packages_dir!r} does not exists") # Also include these packages in the output with --all: wheel, distribute, # setuptools, pip - args = ["pip", "freeze", "--exclude-editable", - "--all", "--path", site_packages_dir] + args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] return subprocess.check_output(args, encoding="utf-8") # noqa: S603 diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 6d5ffdce..4ea1babb 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -243,11 +243,9 @@ def download_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR, repos=tu package = repo.get_package_version(name=name, version=version) if not package: if TRACE_DEEP: - print( - f" download_wheel: No package in {repo.index_url} for {name}=={version}") + print(f" download_wheel: No package in {repo.index_url} for {name}=={version}") continue - supported_wheels = list( - package.get_supported_wheels(environment=environment)) + supported_wheels = list(package.get_supported_wheels(environment=environment)) if not supported_wheels: if TRACE_DEEP: print( @@ -291,8 +289,7 @@ def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): if not package: if TRACE_DEEP: - print( - f" download_sdist: No package in {repo.index_url} for {name}=={version}") + print(f" download_sdist: No package in {repo.index_url} for {name}=={version}") continue sdist = package.sdist if not sdist: @@ -301,8 +298,7 @@ def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): continue if TRACE_DEEP: - print( - f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") + print(f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") fetched_sdist_filename = package.sdist.download(dest_dir=dest_dir) if fetched_sdist_filename: @@ -357,7 +353,6 @@ def sorted(cls, namevers): @attr.attributes class Distribution(NameVer): - # field names that can be updated from another Distribution or mapping updatable_fields = [ "license_expression", @@ -535,8 +530,7 @@ def get_best_download_url(self, repos=tuple()): repos = DEFAULT_PYPI_REPOS for repo in repos: - package = repo.get_package_version( - name=self.name, version=self.version) + package = repo.get_package_version(name=self.name, version=self.version) if not package: if TRACE: print( @@ -776,8 +770,7 @@ def load_remote_about_data(self): if notice_text: about_data["notice_text"] = notice_text except RemoteNotFetchedException: - print( - f"Failed to fetch NOTICE file: {self.notice_download_url}") + print(f"Failed to fetch NOTICE file: {self.notice_download_url}") return self.load_about_data(about_data) def get_checksums(self, dest_dir=THIRDPARTY_DIR): @@ -826,8 +819,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): Fetch license files if missing in `dest_dir`. Return True if license files were fetched. """ - urls = LinksRepository.from_url( - use_cached_index=use_cached_index).links + urls = LinksRepository.from_url(use_cached_index=use_cached_index).links errors = [] extra_lic_names = [lic.get("file") for lic in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] @@ -840,8 +832,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): try: # try remotely first - lic_url = get_license_link_for_filename( - filename=filename, urls=urls) + lic_url = get_license_link_for_filename(filename=filename, urls=urls) fetch_and_save( path_or_url=lic_url, @@ -919,8 +910,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): c for c in classifiers if c.startswith("License") ] license_expression = get_license_expression(declared_license) - other_classifiers = [ - c for c in classifiers if not c.startswith("License")] + other_classifiers = [c for c in classifiers if not c.startswith("License")] holder = raw_data["Author"] holder_contact = raw_data["Author-email"] @@ -962,8 +952,7 @@ def update(self, data, overwrite=False, keep_extra=True): package_url = data.get("package_url") if package_url: purl_from_data = packageurl.PackageURL.from_string(package_url) - purl_from_self = packageurl.PackageURL.from_string( - self.package_url) + purl_from_self = packageurl.PackageURL.from_string(self.package_url) if purl_from_data != purl_from_self: print( f"Invalid dist update attempt, no same same purl with dist: " @@ -1013,8 +1002,7 @@ def get_license_link_for_filename(filename, urls): if not path_or_url: raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: - raise Exception( - f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) + raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) return path_or_url[0] @@ -1102,7 +1090,6 @@ def get_sdist_name_ver_ext(filename): @attr.attributes class Sdist(Distribution): - extension = attr.ib( repr=False, type=str, @@ -1407,8 +1394,7 @@ def packages_from_dir(cls, directory): """ base = os.path.abspath(directory) - paths = [os.path.join(base, f) - for f in os.listdir(base) if f.endswith(EXTENSIONS)] + paths = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] if TRACE_ULTRA_DEEP: print("packages_from_dir: paths:", paths) @@ -1469,8 +1455,7 @@ def dists_from_paths_or_urls(cls, paths_or_urls): dists = [] if TRACE_ULTRA_DEEP: print(" ###paths_or_urls:", paths_or_urls) - installable = [f for f in paths_or_urls if f.endswith( - EXTENSIONS_INSTALLABLE)] + installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)] for path_or_url in installable: try: dist = Distribution.from_path_or_url(path_or_url) @@ -1536,8 +1521,7 @@ class Environment: implementation = attr.ib( type=str, default="cp", - metadata=dict( - help="Python implementation supported by this environment."), + metadata=dict(help="Python implementation supported by this environment."), repr=False, ) @@ -1551,8 +1535,7 @@ class Environment: platforms = attr.ib( type=list, default=attr.Factory(list), - metadata=dict( - help="List of platform tags supported by this environment."), + metadata=dict(help="List of platform tags supported by this environment."), repr=False, ) @@ -1637,8 +1620,7 @@ class PypiSimpleRepository: fetched_package_normalized_names = attr.ib( type=set, default=attr.Factory(set), - metadata=dict( - help="A set of already fetched package normalized names."), + metadata=dict(help="A set of already fetched package normalized names."), ) use_cached_index = attr.ib( @@ -1671,12 +1653,10 @@ def _get_package_versions_map(self, name): self.packages[normalized_name] = versions except RemoteNotFetchedException as e: if TRACE: - print( - f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") + print(f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") if not versions and TRACE: - print( - f"WARNING: package {name} not found in repo: {self.index_url}") + print(f"WARNING: package {name} not found in repo: {self.index_url}") return versions @@ -1861,8 +1841,7 @@ def get(self, path_or_url, as_text=True, force=False): if force or not os.path.exists(cached): if TRACE_DEEP: print(f" FILE CACHE MISS: {path_or_url}") - content = get_file_content( - path_or_url=path_or_url, as_text=as_text) + content = get_file_content(path_or_url=path_or_url, as_text=as_text) wmode = "w" if as_text else "wb" with open(cached, wmode) as fo: fo.write(content) @@ -1884,8 +1863,7 @@ def get_file_content(path_or_url, as_text=True): if path_or_url.startswith("https://"): if TRACE_DEEP: print(f"Fetching: {path_or_url}") - _headers, content = get_remote_file_content( - url=path_or_url, as_text=as_text) + _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) return content elif path_or_url.startswith("file://") or ( @@ -1936,7 +1914,7 @@ def get_remote_file_content( # several redirects and that we can ignore content there. A HEAD request may # not get us this last header print(f" DOWNLOADING: {url}") - with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113 + with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113 status = response.status_code if status != requests.codes.ok: # NOQA if status == 429 and _delay < 20: @@ -1951,8 +1929,7 @@ def get_remote_file_content( ) else: - raise RemoteNotFetchedException( - f"Failed HTTP request from {url} with {status}") + raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") if headers_only: return response.headers, None @@ -2043,8 +2020,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # lets try to get from another dist of the same local package @@ -2056,8 +2032,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get another version of the same package that is not our version @@ -2068,8 +2043,7 @@ def get_other_dists(_package, _dist): ] other_local_version = other_local_packages and other_local_packages[-1] if other_local_version: - latest_local_dists = list( - other_local_version.get_distributions()) + latest_local_dists = list(other_local_version.get_distributions()) for latest_local_dist in latest_local_dists: latest_local_dist.load_about_data(dest_dir=dest_dir) if not latest_local_dist.has_key_metadata(): @@ -2095,8 +2069,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get a latest version of the same package that is not our version @@ -2137,8 +2110,7 @@ def get_other_dists(_package, _dist): # if local_dist.has_key_metadata() or not local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir) - lic_errs = local_dist.fetch_license_files( - dest_dir, use_cached_index=use_cached_index) + lic_errs = local_dist.fetch_license_files(dest_dir, use_cached_index=use_cached_index) if not local_dist.has_key_metadata(): print(f"Unable to add essential ABOUT data for: {local_dist}") @@ -2161,10 +2133,9 @@ def call(args, verbose=TRACE): """ if TRACE_DEEP: print("Calling:", " ".join(args)) - with subprocess.Popen( # noqa: S603 + with subprocess.Popen( # noqa: S603 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" ) as process: - stdouts = [] while True: line = process.stdout.readline() @@ -2260,7 +2231,7 @@ def download_wheels_with_pip( def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603 + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603 except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") @@ -2286,8 +2257,7 @@ def find_problems( for dist in package.get_distributions(): dist.load_about_data(dest_dir=dest_dir) - abpth = os.path.abspath(os.path.join( - dest_dir, dist.about_filename)) + abpth = os.path.abspath(os.path.join(dest_dir, dist.about_filename)) if not dist.has_key_metadata(): print(f" Missing key ABOUT data in file://{abpth}") if "classifiers" in dist.extra_data: From d05665ad44a50b71f66b974ad24c81f7443e8180 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:02:19 +0100 Subject: [PATCH 043/120] Apply cosmetic refactorings Signed-off-by: Philippe Ombredanne --- docs/source/conf.py | 3 ++- etc/scripts/check_thirdparty.py | 4 +--- etc/scripts/fetch_thirdparty.py | 17 ++++++++--------- etc/scripts/gen_pypi_simple.py | 15 +++++++-------- etc/scripts/gen_requirements.py | 4 ++-- etc/scripts/gen_requirements_dev.py | 4 ++-- .../test_utils_pip_compatibility_tags.py | 9 +++++---- etc/scripts/utils_dejacode.py | 9 +++++---- etc/scripts/utils_pip_compatibility_tags.py | 8 +++++--- etc/scripts/utils_requirements.py | 3 +-- etc/scripts/utils_thirdparty.py | 3 ++- 11 files changed, 40 insertions(+), 39 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 8c88fa2c..8aad8294 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -94,7 +94,8 @@ html_show_sphinx = True # Define CSS and HTML abbreviations used in .rst files. These are examples. -# .. role:: is used to refer to styles defined in _static/theme_overrides.css and is used like this: :red:`text` +# .. role:: is used to refer to styles defined in _static/theme_overrides.css +# and is used like this: :red:`text` rst_prolog = """ .. |psf| replace:: Python Software Foundation diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 1aa4e28a..bb8347a5 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -41,8 +40,7 @@ def check_thirdparty_dir( """ Check a thirdparty directory for problems and print these on screen. """ - # check for problems - print(f"==> CHECK FOR PROBLEMS") + print("==> CHECK FOR PROBLEMS") utils_thirdparty.find_problems( dest_dir=dest, report_missing_sources=sdists, diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index c2246837..76a19a60 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -10,7 +9,6 @@ # import itertools -import os import sys from collections import defaultdict @@ -109,7 +107,8 @@ @click.option( "--use-cached-index", is_flag=True, - help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.", + help="Use on disk cached PyPI indexes list of packages and versions and " + "do not refetch if present.", ) @click.option( "--sdist-only", @@ -261,7 +260,7 @@ def fetch_thirdparty( if not fetched: wheels_or_sdist_not_found[f"{name}=={version}"].append(environment) if TRACE: - print(f" NOT FOUND") + print(" NOT FOUND") if sdists or (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only): if TRACE: @@ -276,17 +275,17 @@ def fetch_thirdparty( if not fetched: wheels_or_sdist_not_found[f"{name}=={version}"].append("sdist") if TRACE: - print(f" NOT FOUND") + print(" NOT FOUND") mia = [] for nv, dists in wheels_or_sdist_not_found.items(): name, _, version = nv.partition("==") if name in no_dist: continue - sdist_missing = sdists and "sdist" in dists and not name in wheel_only + sdist_missing = sdists and "sdist" in dists and name not in wheel_only if sdist_missing: mia.append(f"SDist missing: {nv} {dists}") - wheels_missing = wheels and any(d for d in dists if d != "sdist") and not name in sdist_only + wheels_missing = wheels and any(d for d in dists if d != "sdist") and name not in sdist_only if wheels_missing: mia.append(f"Wheels missing: {nv} {dists}") @@ -295,12 +294,12 @@ def fetch_thirdparty( print(m) raise Exception(mia) - print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") + print("==> FETCHING OR CREATING ABOUT AND LICENSE FILES") utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) utils_thirdparty.clean_about_files(dest_dir=dest_dir) # check for problems - print(f"==> CHECK FOR PROBLEMS") + print("==> CHECK FOR PROBLEMS") utils_thirdparty.find_problems( dest_dir=dest_dir, report_missing_sources=sdists, diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index cfe68e67..89d06265 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # SPDX-License-Identifier: BSD-2-Clause-Views AND MIT # Copyright (c) 2010 David Wolever . All rights reserved. @@ -132,7 +131,7 @@ def build_links_package_index(packages_by_package_name, base_url): Return an HTML document as string which is a links index of all packages """ document = [] - header = f""" + header = """ Links for all packages @@ -177,13 +176,13 @@ def simple_index_entry(self, base_url): def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi"): """ - Using a ``directory`` directory of wheels and sdists, create the a PyPI - simple directory index at ``directory``/simple/ populated with the proper - PyPI simple index directory structure crafted using symlinks. + Create the a PyPI simple directory index using a ``directory`` directory of wheels and sdists in + the direvctory at ``directory``/simple/ populated with the proper PyPI simple index directory + structure crafted using symlinks. - WARNING: The ``directory``/simple/ directory is removed if it exists. - NOTE: in addition to the a PyPI simple index.html there is also a links.html - index file generated which is suitable to use with pip's --find-links + WARNING: The ``directory``/simple/ directory is removed if it exists. NOTE: in addition to the a + PyPI simple index.html there is also a links.html index file generated which is suitable to use + with pip's --find-links """ directory = Path(directory) diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py index 2b65ae80..1b879442 100644 --- a/etc/scripts/gen_requirements.py +++ b/etc/scripts/gen_requirements.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -34,7 +33,8 @@ def gen_requirements(): type=pathlib.Path, required=True, metavar="DIR", - help="Path to the 'site-packages' directory where wheels are installed such as lib/python3.6/site-packages", + help="Path to the 'site-packages' directory where wheels are installed " + "such as lib/python3.12/site-packages", ) parser.add_argument( "-r", diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py index 5db1c48e..85482056 100644 --- a/etc/scripts/gen_requirements_dev.py +++ b/etc/scripts/gen_requirements_dev.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -36,7 +35,8 @@ def gen_dev_requirements(): type=pathlib.Path, required=True, metavar="DIR", - help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', + help="Path to the 'site-packages' directory where wheels are installed " + "such as lib/python3.12/site-packages", ) parser.add_argument( "-d", diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py index a33b8b38..de4b7066 100644 --- a/etc/scripts/test_utils_pip_compatibility_tags.py +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -1,4 +1,5 @@ -"""Generate and work with PEP 425 Compatibility Tags. +""" +Generate and work with PEP 425 Compatibility Tags. copied from pip-20.3.1 pip/tests/unit/test_utils_compatibility_tags.py download_url: https://raw.githubusercontent.com/pypa/pip/20.3.1/tests/unit/test_utils_compatibility_tags.py @@ -50,7 +51,7 @@ def test_version_info_to_nodot(version_info, expected): assert actual == expected -class Testcompatibility_tags(object): +class Testcompatibility_tags: def mock_get_config_var(self, **kwd): """ Patch sysconfig.get_config_var for arbitrary keys. @@ -81,7 +82,7 @@ def test_no_hyphen_tag(self): assert "-" not in tag.platform -class TestManylinux2010Tags(object): +class TestManylinux2010Tags: @pytest.mark.parametrize( "manylinux2010,manylinux1", [ @@ -104,7 +105,7 @@ def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): assert arches[:2] == [manylinux2010, manylinux1] -class TestManylinux2014Tags(object): +class TestManylinux2014Tags: @pytest.mark.parametrize( "manylinuxA,manylinuxB", [ diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index cd39cda3..b6bff518 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -25,7 +24,7 @@ DEJACODE_API_URL_PACKAGES = f"{DEJACODE_API_URL}packages/" DEJACODE_API_HEADERS = { - "Authorization": "Token {}".format(DEJACODE_API_KEY), + "Authorization": f"Token {DEJACODE_API_KEY}", "Accept": "application/json; indent=4", } @@ -50,6 +49,7 @@ def fetch_dejacode_packages(params): DEJACODE_API_URL_PACKAGES, params=params, headers=DEJACODE_API_HEADERS, + timeout=10, ) return response.json()["results"] @@ -93,7 +93,7 @@ def update_with_dejacode_about_data(distribution): if package_data: package_api_url = package_data["api_url"] about_url = f"{package_api_url}about" - response = requests.get(about_url, headers=DEJACODE_API_HEADERS) + response = requests.get(about_url, headers=DEJACODE_API_HEADERS, timeout=10) # note that this is YAML-formatted about_text = response.json()["about_data"] about_data = saneyaml.load(about_text) @@ -113,7 +113,7 @@ def fetch_and_save_about_files(distribution, dest_dir="thirdparty"): if package_data: package_api_url = package_data["api_url"] about_url = f"{package_api_url}about_files" - response = requests.get(about_url, headers=DEJACODE_API_HEADERS) + response = requests.get(about_url, headers=DEJACODE_API_HEADERS, timeout=10) about_zip = response.content with io.BytesIO(about_zip) as zf: with zipfile.ZipFile(zf) as zi: @@ -201,6 +201,7 @@ def create_dejacode_package(distribution): DEJACODE_API_URL_PACKAGES, data=new_package_payload, headers=DEJACODE_API_HEADERS, + timeout=10, ) new_package_data = response.json() if response.status_code != 201: diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index de0ac95d..dd954bca 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -1,4 +1,5 @@ -"""Generate and work with PEP 425 Compatibility Tags. +""" +Generate and work with PEP 425 Compatibility Tags. copied from pip-20.3.1 pip/_internal/utils/compatibility_tags.py download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/utils/compatibility_tags.py @@ -130,7 +131,7 @@ def _get_custom_interpreter(implementation=None, version=None): implementation = interpreter_name() if version is None: version = interpreter_version() - return "{}{}".format(implementation, version) + return f"{implementation}{version}" def get_supported( @@ -140,7 +141,8 @@ def get_supported( abis=None, # type: Optional[List[str]] ): # type: (...) -> List[Tag] - """Return a list of supported tags for each version specified in + """ + Return a list of supported tags for each version specified in `versions`. :param version: a string version, of the form "33" or "32", diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 167bc9f5..b9b2c0e7 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -40,7 +39,7 @@ def get_required_name_versions(requirement_lines, with_unpinned=False): req_line = req_line.strip() if not req_line or req_line.startswith("#"): continue - if req_line.startswith("-") or (not with_unpinned and not "==" in req_line): + if req_line.startswith("-") or (not with_unpinned and "==" not in req_line): print(f"Requirement line is not supported: ignored: {req_line}") continue yield get_required_name_version(requirement=req_line, with_unpinned=with_unpinned) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 4ea1babb..aafc1d69 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -91,7 +91,8 @@ - parse requirement file - create a TODO queue of requirements to process -- done: create an empty map of processed binary requirements as {package name: (list of versions/tags} +- done: create an empty map of processed binary requirements as + {package name: (list of versions/tags} - while we have package reqs in TODO queue, process one requirement: From 63bcbf507e8a25f22853d56605c107e47c3673cc Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:05:23 +0100 Subject: [PATCH 044/120] Reformat test code Signed-off-by: Philippe Ombredanne --- .gitignore | 1 + pyproject.toml | 19 +++++++++++-------- tests/test_skeleton_codestyle.py | 25 ++++++++++++++++--------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 2d48196f..8a93c94d 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,4 @@ tcl # Ignore Jupyter Notebook related temp files .ipynb_checkpoints/ +/.ruff_cache/ diff --git a/pyproject.toml b/pyproject.toml index ba55770f..a872ab3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,16 +67,17 @@ include = [ [tool.ruff.lint] # Rules: https://docs.astral.sh/ruff/rules/ select = [ - "E", # pycodestyle - "W", # pycodestyle warnings - "D", # pydocstyle - "F", # Pyflakes - "UP", # pyupgrade - "S", # flake8-bandit +# "E", # pycodestyle +# "W", # pycodestyle warnings +# "D", # pydocstyle +# "F", # Pyflakes +# "UP", # pyupgrade +# "S", # flake8-bandit "I", # isort - "C9", # McCabe complexity +# "C9", # McCabe complexity ] -ignore = ["D1", "D200", "D203", "D205", "D212", "D400", "D415"] +ignore = ["D1", "D200", "D202", "D203", "D205", "D212", "D400", "D415"] + [tool.ruff.lint.isort] force-single-line = true @@ -100,3 +101,5 @@ max-complexity = 10 [tool.ruff.lint.per-file-ignores] # Place paths of files to be ignored by ruff here +"tests/*" = ["S101"] +"test_*.py" = ["S101"] diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py index b4ce8c16..8cd85c94 100644 --- a/tests/test_skeleton_codestyle.py +++ b/tests/test_skeleton_codestyle.py @@ -7,30 +7,37 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import configparser import subprocess import unittest -import configparser - class BaseTests(unittest.TestCase): def test_skeleton_codestyle(self): - """ - This test shouldn't run in proliferated repositories. - """ + # This test shouldn't run in proliferated repositories. + + # TODO: update with switch to pyproject.toml setup_cfg = configparser.ConfigParser() setup_cfg.read("setup.cfg") if setup_cfg["metadata"]["name"] != "skeleton": return - args = "venv/bin/black --check -l 100 setup.py etc tests" + commands = [ + ["venv/bin/ruff", "--check"], + ["venv/bin/ruff", "format", "--check"], + ] + command = None try: - subprocess.check_output(args.split()) + for command in commands: + subprocess.check_output(command) # noqa: S603 except subprocess.CalledProcessError as e: print("===========================================================") print(e.output) print("===========================================================") raise Exception( - "Black style check failed; please format the code using:\n" - " python -m black -l 100 setup.py etc tests", + f"Code style and linting command check failed: {' '.join(command)!r}.\n" + "You can check and format the code using:\n" + " make valid\n", + "OR:\n ruff format\n", + " ruff check --fix\n", e.output, ) from e From 9d1393a85303bf8cf92c9a25aa5cc50bdfd080d1 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:08:25 +0100 Subject: [PATCH 045/120] Format code Signed-off-by: Philippe Ombredanne --- etc/scripts/check_thirdparty.py | 1 + etc/scripts/test_utils_pip_compatibility_tags.py | 1 + tests/test_skeleton_codestyle.py | 1 + 3 files changed, 3 insertions(+) diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index bb8347a5..65ae595e 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -11,6 +11,7 @@ import utils_thirdparty + @click.command() @click.option( "-d", diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py index de4b7066..0e9c360a 100644 --- a/etc/scripts/test_utils_pip_compatibility_tags.py +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -33,6 +33,7 @@ import utils_pip_compatibility_tags + @pytest.mark.parametrize( "version_info, expected", [ diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py index 8cd85c94..7135ac0d 100644 --- a/tests/test_skeleton_codestyle.py +++ b/tests/test_skeleton_codestyle.py @@ -11,6 +11,7 @@ import subprocess import unittest + class BaseTests(unittest.TestCase): def test_skeleton_codestyle(self): # This test shouldn't run in proliferated repositories. From f10b783b6b6fe33032a7862352ed532294efdf14 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:10:45 +0100 Subject: [PATCH 046/120] Refine ruff configuration Signed-off-by: Philippe Ombredanne --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a872ab3a..0f8bd587 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,11 +72,11 @@ select = [ # "D", # pydocstyle # "F", # Pyflakes # "UP", # pyupgrade -# "S", # flake8-bandit + "S", # flake8-bandit "I", # isort # "C9", # McCabe complexity ] -ignore = ["D1", "D200", "D202", "D203", "D205", "D212", "D400", "D415"] +ignore = ["D1", "D200", "D202", "D203", "D205", "D212", "D400", "D415", "I001"] [tool.ruff.lint.isort] From 1d6c8f3bb8755aa7c9d2804240c01b0161417328 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:54:01 +0100 Subject: [PATCH 047/120] Format doc Signed-off-by: Philippe Ombredanne --- AUTHORS.rst | 2 +- README.rst | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 51a19cc8..16e20464 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -1,3 +1,3 @@ The following organizations or individuals have contributed to this repo: -- +- diff --git a/README.rst b/README.rst index 6cbd8395..3d6cb4e1 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,6 @@ A Simple Python Project Skeleton ================================ + This repo attempts to standardize the structure of the Python-based project's repositories using modern Python packaging and configuration techniques. Using this `blog post`_ as inspiration, this repository serves as the base for @@ -47,16 +48,19 @@ Release Notes - 2022-03-04: - Synchronize configure and configure.bat scripts for sanity - Update CI operating system support with latest Azure OS images - - Streamline utility scripts in etc/scripts/ to create, fetch and manage third-party dependencies - There are now fewer scripts. See etc/scripts/README.rst for details + - Streamline utility scripts in etc/scripts/ to create, fetch and manage third-party + dependencies. There are now fewer scripts. See etc/scripts/README.rst for details - 2021-09-03: - - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` and ``requirements-dev.txt`` + - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` + and ``requirements-dev.txt`` - ``configure`` can now accept multiple options at once - Add utility scripts from scancode-toolkit/etc/release/ for use in generating project files - Rename virtual environment directory from ``tmp`` to ``venv`` - - Update README.rst with instructions for generating ``requirements.txt`` and ``requirements-dev.txt``, - as well as collecting dependencies as wheels and generating ABOUT files for them. + - Update README.rst with instructions for generating ``requirements.txt`` + and ``requirements-dev.txt``, as well as collecting dependencies as wheels and generating + ABOUT files for them. - 2021-05-11: - - Adopt new configure scripts from ScanCode TK that allows correct configuration of which Python version is used. + - Adopt new configure scripts from ScanCode TK that allows correct configuration of which + Python version is used. From 0213c1ea9a15ab94a854b8d7af27a1a036e393f4 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:54:35 +0100 Subject: [PATCH 048/120] Run doc8 on all rst files Signed-off-by: Philippe Ombredanne --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 930e8010..debc404e 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ dev: doc8: @echo "-> Run doc8 validation" - @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ + @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ *.rst valid: @echo "-> Run Ruff format" From c112f2a9c20d58e986424f5f32bd259814fc8e3f Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:55:20 +0100 Subject: [PATCH 049/120] Enable doc style checks Signed-off-by: Philippe Ombredanne --- pyproject.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0f8bd587..51761ff8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,8 @@ include = [ "etc/**/*.py", "test/**/*.py", "doc/**/*", - "*.py" + "*.py", + "." ] [tool.ruff.lint] @@ -69,10 +70,10 @@ include = [ select = [ # "E", # pycodestyle # "W", # pycodestyle warnings -# "D", # pydocstyle + "D", # pydocstyle # "F", # Pyflakes # "UP", # pyupgrade - "S", # flake8-bandit +# "S", # flake8-bandit "I", # isort # "C9", # McCabe complexity ] From 944b6c5371bea2ce0763fd26888de6436116d185 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 00:34:26 +0100 Subject: [PATCH 050/120] Add support for new OS versions Signed-off-by: Philippe Ombredanne --- README.rst | 50 +++++++++++++++++++++++++++++++++++++++++---- azure-pipelines.yml | 36 ++++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 6cbd8395..f848b4b3 100644 --- a/README.rst +++ b/README.rst @@ -1,9 +1,11 @@ A Simple Python Project Skeleton ================================ -This repo attempts to standardize the structure of the Python-based project's -repositories using modern Python packaging and configuration techniques. -Using this `blog post`_ as inspiration, this repository serves as the base for -all new Python projects and is mergeable in existing repositories as well. + +This repo attempts to standardize the structure of the Python-based project's repositories using +modern Python packaging and configuration techniques that can then be applied to many repos. + +Using this `blog post`_ as inspiration, this repository serves as the base for all new Python +projects and is mergeable in existing repositories as well. .. _blog post: https://blog.jaraco.com/a-project-skeleton-for-python-projects/ @@ -13,6 +15,7 @@ Usage A brand new project ------------------- + .. code-block:: bash git init my-new-repo @@ -26,6 +29,7 @@ From here, you can make the appropriate changes to the files for your specific p Update an existing project --------------------------- + .. code-block:: bash cd my-existing-project @@ -41,17 +45,54 @@ More usage instructions can be found in ``docs/skeleton-usage.rst``. Release Notes ============= +- 2025-03-29: + + - Add support for beta macOS-15 + - Add support for beta windows-2025 + +- 2025-02-14: + + - Drop support for Python 3.8, add support in CI for Python 3.13, use Python 3.12 as default + version. + +- 2025-01-17: + + - Drop support for macOS-12, add support for macOS-14 + - Add support in CI for ubuntu-24.04 + - Add support in CI for Python 3.12 + +- 2024-08-20: + + - Update references of ownership from nexB to aboutcode-org + +- 2024-07-01: + + - Drop support for Python 3.8 + - Drop support for macOS-11, add support for macOS-14 + +- 2024-02-19: + + - Replace support in CI of default ubuntu-20.04 by ubuntu-22.04 + +- 2023-10-18: + + - Add dark mode support in documentation + - 2023-07-18: + - Add macOS-13 job in azure-pipelines.yml - 2022-03-04: + - Synchronize configure and configure.bat scripts for sanity - Update CI operating system support with latest Azure OS images - Streamline utility scripts in etc/scripts/ to create, fetch and manage third-party dependencies There are now fewer scripts. See etc/scripts/README.rst for details - 2021-09-03: + - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` and ``requirements-dev.txt`` + - ``configure`` can now accept multiple options at once - Add utility scripts from scancode-toolkit/etc/release/ for use in generating project files - Rename virtual environment directory from ``tmp`` to ``venv`` @@ -59,4 +100,5 @@ Release Notes as well as collecting dependencies as wheels and generating ABOUT files for them. - 2021-05-11: + - Adopt new configure scripts from ScanCode TK that allows correct configuration of which Python version is used. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a220f2be..80ae45b1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -26,11 +26,27 @@ jobs: - template: etc/ci/azure-posix.yml parameters: job_name: macos13_cpython + image_name: macOS-13-xlarge + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos13_cpython_arm64 image_name: macOS-13 python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos14_cpython + image_name: macOS-14-large + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + - template: etc/ci/azure-posix.yml parameters: job_name: macos14_cpython_arm64 @@ -41,8 +57,16 @@ jobs: - template: etc/ci/azure-posix.yml parameters: - job_name: macos14_cpython - image_name: macOS-14-large + job_name: macos15_cpython + image_name: macOS-15 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + test_suites: + all: venv/bin/pytest -n 2 -vvs + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos15_cpython_arm64 + image_name: macOS-15-large python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -62,3 +86,11 @@ jobs: python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv\Scripts\pytest -n 2 -vvs + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2025_cpython + image_name: windows-2025 + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + test_suites: + all: venv\Scripts\pytest -n 2 -vvs From 136af3912336616fbd2431a96230961517a2c356 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 12:45:32 +0200 Subject: [PATCH 051/120] Update scripts aboutcode references Signed-off-by: Philippe Ombredanne --- etc/scripts/update_skeleton.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/etc/scripts/update_skeleton.py b/etc/scripts/update_skeleton.py index 635898ba..5705fc43 100644 --- a/etc/scripts/update_skeleton.py +++ b/etc/scripts/update_skeleton.py @@ -1,11 +1,10 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # -# Copyright (c) nexB Inc. and others. All rights reserved. +# Copyright (c) nexB Inc. AboutCode, and others. All rights reserved. # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. +# See https://github.com/aboutcode-org/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # @@ -16,7 +15,7 @@ import click -NEXB_PUBLIC_REPO_NAMES=[ +ABOUTCODE_PUBLIC_REPO_NAMES=[ "aboutcode-toolkit", "ahocode", "bitcode", @@ -56,9 +55,9 @@ @click.command() @click.help_option("-h", "--help") -def update_skeleton_files(repo_names=NEXB_PUBLIC_REPO_NAMES): +def update_skeleton_files(repo_names=ABOUTCODE_PUBLIC_REPO_NAMES): """ - Update project files of nexB projects that use the skeleton + Update project files of AboutCode projects that use the skeleton This script will: - Clone the repo @@ -81,14 +80,14 @@ def update_skeleton_files(repo_names=NEXB_PUBLIC_REPO_NAMES): os.chdir(work_dir_path) # Clone repo - repo_git = f"git@github.com:nexB/{repo_name}.git" + repo_git = f"git@github.com:aboutcode-org/{repo_name}.git" subprocess.run(["git", "clone", repo_git]) # Go into cloned repo os.chdir(work_dir_path / repo_name) # Add skeleton as an origin - subprocess.run(["git", "remote", "add", "skeleton", "git@github.com:nexB/skeleton.git"]) + subprocess.run(["git", "remote", "add", "skeleton", "git@github.com:aboutcode-org/skeleton.git"]) # Fetch skeleton files subprocess.run(["git", "fetch", "skeleton"]) From da8eff0383611df60311b8bac599657450eaeb52 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 14:40:36 +0200 Subject: [PATCH 052/120] Do not format more test data Signed-off-by: Philippe Ombredanne --- pyproject.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 51761ff8..7d807ebf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,10 +60,25 @@ include = [ "src/**/*.py", "etc/**/*.py", "test/**/*.py", + "tests/**/*.py", "doc/**/*", + "docs/**/*", "*.py", "." ] +# ignore test data and testfiles: they should never be linted nor formatted +exclude = [ +# main style + "**/tests/data/**/*", +# scancode-toolkit + "**/tests/*/data/**/*", +# dejacode, purldb + "**/tests/testfiles/**/*", +# vulnerablecode, fetchcode + "**/tests/*/test_data/**/*", + "**/tests/test_data/**/*", +] + [tool.ruff.lint] # Rules: https://docs.astral.sh/ruff/rules/ From 4f9e936d452acc3822df8d3f932cbd7071b31d72 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 14:58:36 +0200 Subject: [PATCH 053/120] Do not treat rst as Python Signed-off-by: Philippe Ombredanne --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7d807ebf..5e16b564 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,8 +61,8 @@ include = [ "etc/**/*.py", "test/**/*.py", "tests/**/*.py", - "doc/**/*", - "docs/**/*", + "doc/**/*.py", + "docs/**/*.py", "*.py", "." ] From a2809fb28c60b54aec0c367285acacdea1cb03a8 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 16:41:57 +0200 Subject: [PATCH 054/120] Combine testing and docs extra for simplicity Signed-off-by: Philippe Ombredanne --- configure | 2 -- configure.bat | 4 ---- setup.cfg | 3 --- 3 files changed, 9 deletions(-) diff --git a/configure b/configure index 22d92885..83fd2035 100755 --- a/configure +++ b/configure @@ -30,7 +30,6 @@ CLI_ARGS=$1 # Requirement arguments passed to pip and used by default or with --dev. REQUIREMENTS="--editable . --constraint requirements.txt" DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" -DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt" # where we create a virtualenv VIRTUALENV_DIR=venv @@ -185,7 +184,6 @@ while getopts :-: optchar; do help ) cli_help;; clean ) find_python && clean;; dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; - docs ) CFG_REQUIREMENTS="$DOCS_REQUIREMENTS";; esac;; esac done diff --git a/configure.bat b/configure.bat index 5b9a9d68..18b37038 100644 --- a/configure.bat +++ b/configure.bat @@ -28,7 +28,6 @@ @rem # Requirement arguments passed to pip and used by default or with --dev. set "REQUIREMENTS=--editable . --constraint requirements.txt" set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" -set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt" @rem # where we create a virtualenv set "VIRTUALENV_DIR=venv" @@ -76,9 +75,6 @@ if not "%1" == "" ( if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" ) - if "%1" EQU "--docs" ( - set "CFG_REQUIREMENTS=%DOCS_REQUIREMENTS%" - ) shift goto again ) diff --git a/setup.cfg b/setup.cfg index aaec643f..ad8e0d8f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,8 +55,6 @@ testing = pycodestyle >= 2.8.0 twine ruff - -docs = Sphinx>=5.0.2 sphinx-rtd-theme>=1.0.0 sphinx-reredirects >= 0.1.2 @@ -64,4 +62,3 @@ docs = sphinx-autobuild sphinx-rtd-dark-mode>=1.3.0 sphinx-copybutton - From 43b96c28baaa1621d24b6f5791c6d915d2edc5f3 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 17:18:19 +0200 Subject: [PATCH 055/120] Refine checking of docs with doc8 Signed-off-by: Philippe Ombredanne --- Makefile | 2 +- pyproject.toml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index debc404e..d21a2f95 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ dev: doc8: @echo "-> Run doc8 validation" - @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ *.rst + @${ACTIVATE} doc8 docs/ *.rst valid: @echo "-> Run Ruff format" diff --git a/pyproject.toml b/pyproject.toml index 5e16b564..bfb1d353 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,3 +119,10 @@ max-complexity = 10 # Place paths of files to be ignored by ruff here "tests/*" = ["S101"] "test_*.py" = ["S101"] + + +[tool.doc8] + +ignore-path = ["docs/build", "doc/build", "docs/_build", "doc/_build"] +max-line-length=100 +verbose=0 From b7194c80c9425087f1d05e430bd9d6a14fb9c3a0 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 18:41:00 +0200 Subject: [PATCH 056/120] Refine doc handling * remove CI scripts and use Makefile targets instead * ensure doc8 runs quiet * add new docs-check make target to run documentation and links checks * update oudated doc for docs contribution Signed-off-by: Philippe Ombredanne --- .github/workflows/docs-ci.yml | 12 +++++------- Makefile | 10 +++++++--- docs/scripts/doc8_style_check.sh | 5 ----- docs/scripts/sphinx_build_link_check.sh | 5 ----- docs/source/conf.py | 2 +- docs/source/contribute/contrib_doc.rst | 8 ++++---- pyproject.toml | 2 -- 7 files changed, 17 insertions(+), 27 deletions(-) delete mode 100755 docs/scripts/doc8_style_check.sh delete mode 100644 docs/scripts/sphinx_build_link_check.sh diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 621de4b2..10ba5faa 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -21,14 +21,12 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Dependencies - run: pip install -e .[docs] + run: ./configure --dev - - name: Check Sphinx Documentation build minimally - working-directory: ./docs - run: sphinx-build -E -W source build + - name: Check documentation and HTML for errors and dead links + run: make docs-check - - name: Check for documentation style errors - working-directory: ./docs - run: ./scripts/doc8_style_check.sh + - name: Check documentation for style errors + run: make doc8 diff --git a/Makefile b/Makefile index d21a2f95..413399e5 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ dev: doc8: @echo "-> Run doc8 validation" - @${ACTIVATE} doc8 docs/ *.rst + @${ACTIVATE} doc8 --quiet docs/ *.rst valid: @echo "-> Run Ruff format" @@ -46,6 +46,10 @@ test: docs: rm -rf docs/_build/ - @${ACTIVATE} sphinx-build docs/ docs/_build/ + @${ACTIVATE} sphinx-build docs/source docs/_build/ -.PHONY: conf dev check valid clean test docs +docs-check: + @${ACTIVATE} sphinx-build -E -W -b html docs/source docs/_build/ + @${ACTIVATE} sphinx-build -E -W -b linkcheck docs/source docs/_build/ + +.PHONY: conf dev check valid clean test docs docs-check diff --git a/docs/scripts/doc8_style_check.sh b/docs/scripts/doc8_style_check.sh deleted file mode 100755 index 94163239..00000000 --- a/docs/scripts/doc8_style_check.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# halt script on error -set -e -# Check for Style Code Violations -doc8 --max-line-length 100 source --ignore D000 --quiet \ No newline at end of file diff --git a/docs/scripts/sphinx_build_link_check.sh b/docs/scripts/sphinx_build_link_check.sh deleted file mode 100644 index c5426863..00000000 --- a/docs/scripts/sphinx_build_link_check.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# halt script on error -set -e -# Build locally, and then check links -sphinx-build -E -W -b linkcheck source build \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 8aad8294..056ca6ea 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -18,7 +18,7 @@ # -- Project information ----------------------------------------------------- project = "nexb-skeleton" -copyright = "nexB Inc. and others." +copyright = "nexB Inc., AboutCode and others." author = "AboutCode.org authors and contributors" diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst index 5640db26..041b3583 100644 --- a/docs/source/contribute/contrib_doc.rst +++ b/docs/source/contribute/contrib_doc.rst @@ -147,7 +147,7 @@ What is Checked? ^^^^^^^^^^^^^^^^ PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. -Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. +Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. What is checked: @@ -169,11 +169,11 @@ What is checked: Interspinx ---------- -ScanCode toolkit documentation uses `Intersphinx `_ +ScanCode toolkit documentation uses `Intersphinx `_ to link to other Sphinx Documentations, to maintain links to other Aboutcode Projects. To link sections in the same documentation, standart reST labels are used. Refer -`Cross-Referencing `_ for more information. +`Cross-Referencing `_ for more information. For example:: @@ -230,7 +230,7 @@ Style Conventions for the Documentaion 1. Headings - (`Refer `_) + (`Refer `_) Normally, there are no heading levels assigned to certain characters as the structure is determined from the succession of headings. However, this convention is used in Python’s Style Guide for documenting which you may follow: diff --git a/pyproject.toml b/pyproject.toml index bfb1d353..c9e67720 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,7 +122,5 @@ max-complexity = 10 [tool.doc8] - ignore-path = ["docs/build", "doc/build", "docs/_build", "doc/_build"] max-line-length=100 -verbose=0 From a5bcdbdd71d1542a0e9ec9b190a2e3d573c53744 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 18:49:01 +0200 Subject: [PATCH 057/120] Add twine check to release publication Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index a66c9c80..cf0579a7 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -30,12 +30,15 @@ jobs: with: python-version: 3.12 - - name: Install pypa/build - run: python -m pip install build --user + - name: Install pypa/build and twine + run: python -m pip install --user build twine - name: Build a binary wheel and a source tarball run: python -m build --sdist --wheel --outdir dist/ + - name: Validate wheel and sdis for Pypi + run: python -m twine check dist/* + - name: Upload built archives uses: actions/upload-artifact@v4 with: From a6c25fb2a2fa35311d26621b9db400ca52bd376e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 19:16:31 +0200 Subject: [PATCH 058/120] Refine doc contribution docs Signed-off-by: Philippe Ombredanne --- docs/source/contribute/contrib_doc.rst | 119 ++++++++----------------- 1 file changed, 38 insertions(+), 81 deletions(-) diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst index 041b3583..dee9296d 100644 --- a/docs/source/contribute/contrib_doc.rst +++ b/docs/source/contribute/contrib_doc.rst @@ -8,109 +8,59 @@ Contributing to the Documentation Setup Local Build ----------------- -To get started, create or identify a working directory on your local machine. +To get started, check out and configure the repository for development:: -Open that directory and execute the following command in a terminal session:: + git clone https://github.com/aboutcode-org/.git - git clone https://github.com/aboutcode-org/skeleton.git + cd your-repo + ./configure --dev -That will create an ``/skeleton`` directory in your working directory. -Now you can install the dependencies in a virtualenv:: - - cd skeleton - ./configure --docs +(Or use "make dev") .. note:: - In case of windows, run ``configure --docs`` instead of this. - -Now, this will install the following prerequisites: - -- Sphinx -- sphinx_rtd_theme (the format theme used by ReadTheDocs) -- docs8 (style linter) + In case of windows, run ``configure --dev``. -These requirements are already present in setup.cfg and `./configure --docs` installs them. +This will install and configure all requirements foer development including for docs development. -Now you can build the HTML documents locally:: +Now you can build the HTML documentation locally:: source venv/bin/activate - cd docs - make html - -Assuming that your Sphinx installation was successful, Sphinx should build a local instance of the -documentation .html files:: - - open build/html/index.html - -.. note:: - - In case this command did not work, for example on Ubuntu 18.04 you may get a message like “Couldn’t - get a file descriptor referring to the console”, try: - - :: - - see build/html/index.html + make docs -You now have a local build of the AboutCode documents. +This will build a local instance of the ``docs/_build`` directory:: -.. _contrib_doc_share_improvements: + open docs/_build/index.html -Share Document Improvements ---------------------------- - -Ensure that you have the latest files:: - - git pull - git status -Before commiting changes run Continious Integration Scripts locally to run tests. Refer -:ref:`doc_ci` for instructions on the same. +To validate the documentation style and content, use:: -Follow standard git procedures to upload your new and modified files. The following commands are -examples:: - - git status - git add source/index.rst - git add source/how-to-scan.rst - git status - git commit -m "New how-to document that explains how to scan" - git status - git push - git status - -The Scancode-Toolkit webhook with ReadTheDocs should rebuild the documentation after your -Pull Request is Merged. + source venv/bin/activate + make doc8 + make docs-check -Refer the `Pro Git Book `_ available online for Git tutorials -covering more complex topics on Branching, Merging, Rebasing etc. .. _doc_ci: Continuous Integration ---------------------- -The documentations are checked on every new commit through Travis-CI, so that common errors are -avoided and documentation standards are enforced. Travis-CI presently checks for these 3 aspects -of the documentation : +The documentations are checked on every new commit, so that common errors are avoided and +documentation standards are enforced. We checks for these aspects of the documentation: 1. Successful Builds (By using ``sphinx-build``) -2. No Broken Links (By Using ``link-check``) -3. Linting Errors (By Using ``Doc8``) +2. No Broken Links (By Using ``linkcheck``) +3. Linting Errors (By Using ``doc8``) -So run these scripts at your local system before creating a Pull Request:: +You myst run these scripts locally before creating a pull request:: - cd docs - ./scripts/sphinx_build_link_check.sh - ./scripts/doc8_style_check.sh + make doc8 + make check-docs -If you don't have permission to run the scripts, run:: - - chmod u+x ./scripts/doc8_style_check.sh .. _doc_style_docs8: -Style Checks Using ``Doc8`` +Style Checks Using ``doc8`` --------------------------- How To Run Style Tests @@ -118,8 +68,7 @@ How To Run Style Tests In the project root, run the following commands:: - $ cd docs - $ ./scripts/doc8_style_check.sh + make doc8 A sample output is:: @@ -143,11 +92,13 @@ A sample output is:: Now fix the errors and run again till there isn't any style error in the documentation. + What is Checked? ^^^^^^^^^^^^^^^^ PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. -Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. +Doc8 is a sub-project of the same Organization. Refer this +`README `_ for more details. What is checked: @@ -164,16 +115,19 @@ What is checked: - no carriage returns (use UNIX newlines) - D004 - no newline at end of file - D005 + .. _doc_interspinx: Interspinx ---------- -ScanCode toolkit documentation uses `Intersphinx `_ +AboutCode documentation uses +`Intersphinx `_ to link to other Sphinx Documentations, to maintain links to other Aboutcode Projects. To link sections in the same documentation, standart reST labels are used. Refer -`Cross-Referencing `_ for more information. +`Cross-Referencing `_ +for more information. For example:: @@ -223,6 +177,7 @@ Intersphinx, and you link to that label, it will create a link to the local labe For more information, refer this tutorial named `Using Intersphinx `_. + .. _doc_style_conv: Style Conventions for the Documentaion @@ -303,12 +258,14 @@ Style Conventions for the Documentaion ``rst_snippets/warning_snippets/`` and then included to eliminate redundancy, as these are frequently used in multiple files. + Converting from Markdown ------------------------ -If you want to convert a ``.md`` file to a ``.rst`` file, this `tool `_ -does it pretty well. You'd still have to clean up and check for errors as this contains a lot of -bugs. But this is definitely better than converting everything by yourself. +If you want to convert a ``.md`` file to a ``.rst`` file, this +`tool `_ does it pretty well. +You will still have to clean up and check for errors as this contains a lot of bugs. But this is +definitely better than converting everything by yourself. This will be helpful in converting GitHub wiki's (Markdown Files) to reStructuredtext files for Sphinx/ReadTheDocs hosting. From 68daae1e7e475a89568e353f64f29af13754ce9e Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 27 Mar 2025 14:54:31 -0700 Subject: [PATCH 059/120] Replace black and isort with ruff * Use ruff config and Make commands from scancode.io Signed-off-by: Jono Yang --- Makefile | 27 ++++++++++++--------------- pyproject.toml | 37 +++++++++++++++++++++++++++++++++++++ setup.cfg | 3 +-- 3 files changed, 50 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 94451b33..1738b20a 100644 --- a/Makefile +++ b/Makefile @@ -17,27 +17,24 @@ dev: @echo "-> Configure the development envt." ./configure --dev -isort: - @echo "-> Apply isort changes to ensure proper imports ordering" - ${VENV}/bin/isort --sl -l 100 src tests setup.py - -black: - @echo "-> Apply black code formatter" - ${VENV}/bin/black -l 100 src tests setup.py - doc8: @echo "-> Run doc8 validation" @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ -valid: isort black +valid: + @echo "-> Run Ruff format" + @${ACTIVATE} ruff format + @echo "-> Run Ruff linter" + @${ACTIVATE} ruff check --fix check: - @echo "-> Run pycodestyle (PEP8) validation" - @${ACTIVATE} pycodestyle --max-line-length=100 --exclude=.eggs,venv,lib,thirdparty,docs,migrations,settings.py,.cache . - @echo "-> Run isort imports ordering validation" - @${ACTIVATE} isort --sl --check-only -l 100 setup.py src tests . - @echo "-> Run black validation" - @${ACTIVATE} black --check --check -l 100 src tests setup.py + @echo "-> Run Ruff linter validation (pycodestyle, bandit, isort, and more)" + @${ACTIVATE} ruff check + @echo "-> Run Ruff format validation" + @${ACTIVATE} ruff format --check + @$(MAKE) doc8 + @echo "-> Run ABOUT files validation" + @${ACTIVATE} about check etc/ clean: @echo "-> Clean the Python env" diff --git a/pyproject.toml b/pyproject.toml index cde79074..01e60fc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,3 +50,40 @@ addopts = [ "--strict-markers", "--doctest-modules" ] + +[tool.ruff] +line-length = 88 +extend-exclude = [] +target-version = "py310" + +[tool.ruff.lint] +# Rules: https://docs.astral.sh/ruff/rules/ +select = [ + "E", # pycodestyle + "W", # pycodestyle warnings + "D", # pydocstyle + "F", # Pyflakes + "UP", # pyupgrade + "S", # flake8-bandit + "I", # isort + "C9", # McCabe complexity +] +ignore = ["D1", "D203", "D205", "D212", "D400", "D415"] + +[tool.ruff.lint.isort] +force-single-line = true +sections = { django = ["django"] } +section-order = [ + "future", + "standard-library", + "django", + "third-party", + "first-party", + "local-folder", +] + +[tool.ruff.lint.mccabe] +max-complexity = 10 + +[tool.ruff.lint.per-file-ignores] +# Place paths of files to be ignored by ruff here diff --git a/setup.cfg b/setup.cfg index ef7d369b..aaec643f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,8 +54,7 @@ testing = aboutcode-toolkit >= 7.0.2 pycodestyle >= 2.8.0 twine - black - isort + ruff docs = Sphinx>=5.0.2 From 6a8c9ae144a1985b59fb69a0b2c55e32831714b8 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 00:46:06 +0100 Subject: [PATCH 060/120] Use org standard 100 line length Signed-off-by: Philippe Ombredanne --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 01e60fc6..cea91bd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ addopts = [ ] [tool.ruff] -line-length = 88 +line-length = 100 extend-exclude = [] target-version = "py310" From 2fd31d54afa47418c764de0f1a30d67c7059ed7b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 08:40:28 +0100 Subject: [PATCH 061/120] Lint all common code directories Signed-off-by: Philippe Ombredanne --- pyproject.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index cea91bd1..9e627366 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,14 @@ addopts = [ line-length = 100 extend-exclude = [] target-version = "py310" +include = [ + "pyproject.toml", + "src/**/*.py", + "etc/**/*.py", + "test/**/*.py", + "doc/**/*", + "*.py" +] [tool.ruff.lint] # Rules: https://docs.astral.sh/ruff/rules/ From eb5fc82ab1cab8a4742a2b9028d1436956960e81 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 09:07:47 +0100 Subject: [PATCH 062/120] Remove unused targets Signed-off-by: Philippe Ombredanne --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1738b20a..930e8010 100644 --- a/Makefile +++ b/Makefile @@ -48,4 +48,4 @@ docs: rm -rf docs/_build/ @${ACTIVATE} sphinx-build docs/ docs/_build/ -.PHONY: conf dev check valid black isort clean test docs +.PHONY: conf dev check valid clean test docs From 529d51621c9e2af8e1ec2503044b8752c71c3ba7 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 11:03:05 +0100 Subject: [PATCH 063/120] Improve import sorting Signed-off-by: Philippe Ombredanne --- etc/scripts/check_thirdparty.py | 1 - etc/scripts/fetch_thirdparty.py | 2 +- etc/scripts/test_utils_pip_compatibility_tags.py | 3 +-- etc/scripts/utils_dejacode.py | 1 - etc/scripts/utils_pip_compatibility_tags.py | 14 ++++++-------- etc/scripts/utils_thirdparty.py | 3 +-- pyproject.toml | 7 ++++++- 7 files changed, 15 insertions(+), 16 deletions(-) diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 2daded94..62dbb14f 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -12,7 +12,6 @@ import utils_thirdparty - @click.command() @click.option( "-d", diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 3f9ff527..30d376c4 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -16,8 +16,8 @@ import click -import utils_thirdparty import utils_requirements +import utils_thirdparty TRACE = False TRACE_DEEP = False diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py index 98187c56..a33b8b38 100644 --- a/etc/scripts/test_utils_pip_compatibility_tags.py +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -25,14 +25,13 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -from unittest.mock import patch import sysconfig +from unittest.mock import patch import pytest import utils_pip_compatibility_tags - @pytest.mark.parametrize( "version_info, expected", [ diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index 652252d4..c71543f6 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -14,7 +14,6 @@ import requests import saneyaml - from packvers import version as packaging_version """ diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index af42a0cd..de0ac95d 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -27,14 +27,12 @@ import re -from packvers.tags import ( - compatible_tags, - cpython_tags, - generic_tags, - interpreter_name, - interpreter_version, - mac_platforms, -) +from packvers.tags import compatible_tags +from packvers.tags import cpython_tags +from packvers.tags import generic_tags +from packvers.tags import interpreter_name +from packvers.tags import interpreter_version +from packvers.tags import mac_platforms _osx_arch_pat = re.compile(r"(.+)_(\d+)_(\d+)_(.+)") diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 46dc7289..b0295eca 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -25,14 +25,13 @@ import packageurl import requests import saneyaml +import utils_pip_compatibility_tags from commoncode import fileutils from commoncode.hash import multi_checksums from commoncode.text import python_safe_name from packvers import tags as packaging_tags from packvers import version as packaging_version -import utils_pip_compatibility_tags - """ Utilities to manage Python thirparty libraries source, binaries and metadata in local directories and remote repositories. diff --git a/pyproject.toml b/pyproject.toml index 9e627366..ba55770f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,10 +76,15 @@ select = [ "I", # isort "C9", # McCabe complexity ] -ignore = ["D1", "D203", "D205", "D212", "D400", "D415"] +ignore = ["D1", "D200", "D203", "D205", "D212", "D400", "D415"] [tool.ruff.lint.isort] force-single-line = true +lines-after-imports = 1 +default-section = "first-party" +known-first-party = ["src", "tests", "etc/scripts/**/*.py"] +known-third-party = ["click", "pytest"] + sections = { django = ["django"] } section-order = [ "future", From aae1a2847c0e493b0e8bea542da30dbdfb2be68e Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 21:35:16 +0100 Subject: [PATCH 064/120] Apply small code updates Signed-off-by: Philippe Ombredanne --- etc/scripts/utils_requirements.py | 20 ++++++++----- etc/scripts/utils_thirdparty.py | 48 +++++++++++++++---------------- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 1c502390..a9ac2235 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -57,21 +57,25 @@ def get_required_name_version(requirement, with_unpinned=False): >>> assert get_required_name_version("fooA==1.2.3.DEV1") == ("fooa", "1.2.3.dev1") >>> assert get_required_name_version("foo==1.2.3", with_unpinned=False) == ("foo", "1.2.3") >>> assert get_required_name_version("foo", with_unpinned=True) == ("foo", "") - >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == ("foo", ""), get_required_name_version("foo>=1.2") + >>> expected = ("foo", ""), get_required_name_version("foo>=1.2") + >>> assert get_required_name_version("foo>=1.2", with_unpinned=True) == expected >>> try: ... assert not get_required_name_version("foo", with_unpinned=False) ... except Exception as e: ... assert "Requirement version must be pinned" in str(e) """ requirement = requirement and "".join(requirement.lower().split()) - assert requirement, f"specifier is required is empty:{requirement!r}" + if not requirement: + raise ValueError(f"specifier is required is empty:{requirement!r}") name, operator, version = split_req(requirement) - assert name, f"Name is required: {requirement}" + if not name: + raise ValueError(f"Name is required: {requirement}") is_pinned = operator == "==" if with_unpinned: version = "" else: - assert is_pinned and version, f"Requirement version must be pinned: {requirement}" + if not is_pinned and version: + raise ValueError(f"Requirement version must be pinned: {requirement}") return name, version @@ -120,7 +124,7 @@ def get_installed_reqs(site_packages_dir): # setuptools, pip args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] - return subprocess.check_output(args, encoding="utf-8") + return subprocess.check_output(args, encoding="utf-8") # noqa: S603 comparators = ( @@ -150,9 +154,11 @@ def split_req(req): >>> assert split_req("foo >= 1.2.3 ") == ("foo", ">=", "1.2.3"), split_req("foo >= 1.2.3 ") >>> assert split_req("foo>=1.2") == ("foo", ">=", "1.2"), split_req("foo>=1.2") """ - assert req + if not req: + raise ValueError("req is required") # do not allow multiple constraints and tags - assert not any(c in req for c in ",;") + if not any(c in req for c in ",;"): + raise Exception(f"complex requirements with : or ; not supported: {req}") req = "".join(req.split()) if not any(c in req for c in comparators): return req, "", "" diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index b0295eca..6d5ffdce 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -559,7 +558,8 @@ def download(self, dest_dir=THIRDPARTY_DIR): Download this distribution into `dest_dir` directory. Return the fetched filename. """ - assert self.filename + if not self.filename: + raise ValueError(f"self.filename has no value but is required: {self.filename!r}") if TRACE_DEEP: print( f"Fetching distribution of {self.name}=={self.version}:", @@ -829,10 +829,9 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): urls = LinksRepository.from_url( use_cached_index=use_cached_index).links errors = [] - extra_lic_names = [l.get("file") - for l in self.extra_data.get("licenses", {})] + extra_lic_names = [lic.get("file") for lic in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] - extra_lic_names = [ln for ln in extra_lic_names if ln] + extra_lic_names = [eln for eln in extra_lic_names if eln] lic_names = [f"{key}.LICENSE" for key in self.get_license_keys()] for filename in lic_names + extra_lic_names: floc = os.path.join(dest_dir, filename) @@ -853,7 +852,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from remote: {lic_url}") - except: + except Exception: try: # try licensedb second lic_url = f"{LICENSEDB_API_URL}/{filename}" @@ -866,8 +865,9 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): if TRACE: print(f"Fetched license from licensedb: {lic_url}") - except: - msg = f'No text for license {filename} in expression "{self.license_expression}" from {self}' + except Exception: + msg = f"No text for license {filename} in expression " + f"{self.license_expression!r} from {self}" print(msg) errors.append(msg) @@ -1009,7 +1009,7 @@ def get_license_link_for_filename(filename, urls): exception if no link is found or if there are more than one link for that file name. """ - path_or_url = [l for l in urls if l.endswith(f"/{filename}")] + path_or_url = [url for url in urls if url.endswith(f"/{filename}")] if not path_or_url: raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: @@ -1140,7 +1140,6 @@ def to_filename(self): @attr.attributes class Wheel(Distribution): - """ Represents a wheel file. @@ -1301,7 +1300,7 @@ def is_pure(self): def is_pure_wheel(filename): try: return Wheel.from_filename(filename).is_pure() - except: + except Exception: return False @@ -1489,8 +1488,7 @@ def dists_from_paths_or_urls(cls, paths_or_urls): ) except InvalidDistributionFilename: if TRACE_DEEP: - print( - f" Skipping invalid distribution from: {path_or_url}") + print(f" Skipping invalid distribution from: {path_or_url}") continue return dists @@ -1500,8 +1498,7 @@ def get_distributions(self): """ if self.sdist: yield self.sdist - for wheel in self.wheels: - yield wheel + yield from self.wheels def get_url_for_filename(self, filename): """ @@ -1632,7 +1629,8 @@ class PypiSimpleRepository: type=dict, default=attr.Factory(lambda: defaultdict(dict)), metadata=dict( - help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} available in this repo" + help="Mapping of {name: {version: PypiPackage, version: PypiPackage, etc} " + "available in this repo" ), ) @@ -1647,7 +1645,8 @@ class PypiSimpleRepository: type=bool, default=False, metadata=dict( - help="If True, use any existing on-disk cached PyPI index files. Otherwise, fetch and cache." + help="If True, use any existing on-disk cached PyPI index files. " + "Otherwise, fetch and cache." ), ) @@ -1656,7 +1655,8 @@ def _get_package_versions_map(self, name): Return a mapping of all available PypiPackage version for this package name. The mapping may be empty. It is ordered by version from oldest to newest """ - assert name + if not name: + raise ValueError(f"name is required: {name!r}") normalized_name = NameVer.normalize_name(name) versions = self.packages[normalized_name] if not versions and normalized_name not in self.fetched_package_normalized_names: @@ -1713,7 +1713,7 @@ def fetch_links(self, normalized_name): ) links = collect_urls(text) # TODO: keep sha256 - links = [l.partition("#sha256=") for l in links] + links = [link.partition("#sha256=") for link in links] links = [url for url, _, _sha256 in links] return links @@ -1936,7 +1936,7 @@ def get_remote_file_content( # several redirects and that we can ignore content there. A HEAD request may # not get us this last header print(f" DOWNLOADING: {url}") - with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: + with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113 status = response.status_code if status != requests.codes.ok: # NOQA if status == 429 and _delay < 20: @@ -2161,7 +2161,7 @@ def call(args, verbose=TRACE): """ if TRACE_DEEP: print("Calling:", " ".join(args)) - with subprocess.Popen( + with subprocess.Popen( # noqa: S603 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" ) as process: @@ -2227,7 +2227,7 @@ def download_wheels_with_pip( cli_args.extend(["--requirement", req_file]) if TRACE: - print(f"Downloading wheels using command:", " ".join(cli_args)) + print("Downloading wheels using command:", " ".join(cli_args)) existing = set(os.listdir(dest_dir)) error = False @@ -2260,7 +2260,7 @@ def download_wheels_with_pip( def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603 except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") @@ -2312,5 +2312,5 @@ def get_license_expression(declared_licenses): return get_only_expression_from_extracted_license(declared_licenses) except ImportError: # Scancode is not installed, clean and join all the licenses - lics = [python_safe_name(l).lower() for l in declared_licenses] + lics = [python_safe_name(lic).lower() for lic in declared_licenses] return " AND ".join(lics).lower() From 037f9fc1b03736eeac9e0eefac3e35acc916d193 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 21:42:03 +0100 Subject: [PATCH 065/120] Format code Signed-off-by: Philippe Ombredanne --- etc/scripts/check_thirdparty.py | 3 +- etc/scripts/fetch_thirdparty.py | 26 ++++----- etc/scripts/gen_pypi_simple.py | 4 +- etc/scripts/utils_dejacode.py | 15 +++--- etc/scripts/utils_requirements.py | 9 ++-- etc/scripts/utils_thirdparty.py | 90 +++++++++++-------------------- 6 files changed, 50 insertions(+), 97 deletions(-) diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 62dbb14f..1aa4e28a 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -16,8 +16,7 @@ @click.option( "-d", "--dest", - type=click.Path(exists=True, readable=True, - path_type=str, file_okay=False), + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, help="Path to the thirdparty directory to check.", ) diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index 30d376c4..c2246837 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -55,8 +55,7 @@ "-d", "--dest", "dest_dir", - type=click.Path(exists=True, readable=True, - path_type=str, file_okay=False), + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), metavar="DIR", default=utils_thirdparty.THIRDPARTY_DIR, show_default=True, @@ -121,7 +120,7 @@ show_default=False, multiple=True, help="Package name(s) that come only in sdist format (no wheels). " - "The command will not fail and exit if no wheel exists for these names", + "The command will not fail and exit if no wheel exists for these names", ) @click.option( "--wheel-only", @@ -132,7 +131,7 @@ show_default=False, multiple=True, help="Package name(s) that come only in wheel format (no sdist). " - "The command will not fail and exit if no sdist exists for these names", + "The command will not fail and exit if no sdist exists for these names", ) @click.option( "--no-dist", @@ -143,7 +142,7 @@ show_default=False, multiple=True, help="Package name(s) that do not come either in wheel or sdist format. " - "The command will not fail and exit if no distribution exists for these names", + "The command will not fail and exit if no distribution exists for these names", ) @click.help_option("-h", "--help") def fetch_thirdparty( @@ -225,8 +224,7 @@ def fetch_thirdparty( environments = None if wheels: evts = itertools.product(python_versions, operating_systems) - environments = [utils_thirdparty.Environment.from_pyver_and_os( - pyv, os) for pyv, os in evts] + environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] # Collect PyPI repos repos = [] @@ -250,7 +248,6 @@ def fetch_thirdparty( print(f"Processing: {name} @ {version}") if wheels: for environment in environments: - if TRACE: print(f" ==> Fetching wheel for envt: {environment}") @@ -262,14 +259,11 @@ def fetch_thirdparty( repos=repos, ) if not fetched: - wheels_or_sdist_not_found[f"{name}=={version}"].append( - environment) + wheels_or_sdist_not_found[f"{name}=={version}"].append(environment) if TRACE: print(f" NOT FOUND") - if (sdists or - (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only) - ): + if sdists or (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only): if TRACE: print(f" ==> Fetching sdist: {name}=={version}") @@ -292,8 +286,7 @@ def fetch_thirdparty( sdist_missing = sdists and "sdist" in dists and not name in wheel_only if sdist_missing: mia.append(f"SDist missing: {nv} {dists}") - wheels_missing = wheels and any( - d for d in dists if d != "sdist") and not name in sdist_only + wheels_missing = wheels and any(d for d in dists if d != "sdist") and not name in sdist_only if wheels_missing: mia.append(f"Wheels missing: {nv} {dists}") @@ -303,8 +296,7 @@ def fetch_thirdparty( raise Exception(mia) print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") - utils_thirdparty.fetch_abouts_and_licenses( - dest_dir=dest_dir, use_cached_index=use_cached_index) + utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) utils_thirdparty.clean_about_files(dest_dir=dest_dir) # check for problems diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 214d90dc..cfe68e67 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -69,7 +69,6 @@ def get_package_name_from_filename(filename): raise InvalidDistributionFilename(filename) elif filename.endswith(wheel_ext): - wheel_info = get_wheel_from_filename(filename) if not wheel_info: @@ -200,11 +199,10 @@ def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi" simple_html_index = [ "", "PyPI Simple Index", - '' '', + '', ] for pkg_file in directory.iterdir(): - pkg_filename = pkg_file.name if ( diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index c71543f6..cd39cda3 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -32,8 +32,7 @@ def can_do_api_calls(): if not DEJACODE_API_KEY and DEJACODE_API_URL: - print( - "DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") + print("DejaCode DEJACODE_API_KEY and DEJACODE_API_URL not configured. Doing nothing") return False else: return True @@ -68,8 +67,7 @@ def get_package_data(distribution): return results[0] elif len_results > 1: - print( - f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") + print(f"More than 1 entry exists, review at: {DEJACODE_API_URL_PACKAGES}") else: print("Could not find package:", distribution.download_url) @@ -150,12 +148,11 @@ def find_latest_dejacode_package(distribution): # there was no exact match, find the latest version # TODO: consider the closest version rather than the latest # or the version that has the best data - with_versions = [(packaging_version.parse(p["version"]), p) - for p in packages] + with_versions = [(packaging_version.parse(p["version"]), p) for p in packages] with_versions = sorted(with_versions) latest_version, latest_package_version = sorted(with_versions)[-1] print( - f"Found DejaCode latest version: {latest_version} " f"for dist: {distribution.package_url}", + f"Found DejaCode latest version: {latest_version} for dist: {distribution.package_url}", ) return latest_package_version @@ -181,7 +178,7 @@ def create_dejacode_package(distribution): } fields_to_carry_over = [ - "download_url" "type", + "download_urltype", "namespace", "name", "version", @@ -209,5 +206,5 @@ def create_dejacode_package(distribution): if response.status_code != 201: raise Exception(f"Error, cannot create package for: {distribution}") - print(f'New Package created at: {new_package_data["absolute_url"]}') + print(f"New Package created at: {new_package_data['absolute_url']}") return new_package_data diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index a9ac2235..167bc9f5 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -106,8 +106,7 @@ def lock_dev_requirements( all_req_nvs = get_required_name_versions(all_req_lines) dev_only_req_nvs = {n: v for n, v in all_req_nvs if n not in main_names} - new_reqs = "\n".join( - f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) + new_reqs = "\n".join(f"{n}=={v}" for n, v in sorted(dev_only_req_nvs.items())) with open(dev_requirements_file, "w") as fo: fo.write(new_reqs) @@ -118,12 +117,10 @@ def get_installed_reqs(site_packages_dir): as a text. """ if not os.path.exists(site_packages_dir): - raise Exception( - f"site_packages directory: {site_packages_dir!r} does not exists") + raise Exception(f"site_packages directory: {site_packages_dir!r} does not exists") # Also include these packages in the output with --all: wheel, distribute, # setuptools, pip - args = ["pip", "freeze", "--exclude-editable", - "--all", "--path", site_packages_dir] + args = ["pip", "freeze", "--exclude-editable", "--all", "--path", site_packages_dir] return subprocess.check_output(args, encoding="utf-8") # noqa: S603 diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 6d5ffdce..4ea1babb 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -243,11 +243,9 @@ def download_wheel(name, version, environment, dest_dir=THIRDPARTY_DIR, repos=tu package = repo.get_package_version(name=name, version=version) if not package: if TRACE_DEEP: - print( - f" download_wheel: No package in {repo.index_url} for {name}=={version}") + print(f" download_wheel: No package in {repo.index_url} for {name}=={version}") continue - supported_wheels = list( - package.get_supported_wheels(environment=environment)) + supported_wheels = list(package.get_supported_wheels(environment=environment)) if not supported_wheels: if TRACE_DEEP: print( @@ -291,8 +289,7 @@ def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): if not package: if TRACE_DEEP: - print( - f" download_sdist: No package in {repo.index_url} for {name}=={version}") + print(f" download_sdist: No package in {repo.index_url} for {name}=={version}") continue sdist = package.sdist if not sdist: @@ -301,8 +298,7 @@ def download_sdist(name, version, dest_dir=THIRDPARTY_DIR, repos=tuple()): continue if TRACE_DEEP: - print( - f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") + print(f" download_sdist: Getting sdist from index (or cache): {sdist.download_url}") fetched_sdist_filename = package.sdist.download(dest_dir=dest_dir) if fetched_sdist_filename: @@ -357,7 +353,6 @@ def sorted(cls, namevers): @attr.attributes class Distribution(NameVer): - # field names that can be updated from another Distribution or mapping updatable_fields = [ "license_expression", @@ -535,8 +530,7 @@ def get_best_download_url(self, repos=tuple()): repos = DEFAULT_PYPI_REPOS for repo in repos: - package = repo.get_package_version( - name=self.name, version=self.version) + package = repo.get_package_version(name=self.name, version=self.version) if not package: if TRACE: print( @@ -776,8 +770,7 @@ def load_remote_about_data(self): if notice_text: about_data["notice_text"] = notice_text except RemoteNotFetchedException: - print( - f"Failed to fetch NOTICE file: {self.notice_download_url}") + print(f"Failed to fetch NOTICE file: {self.notice_download_url}") return self.load_about_data(about_data) def get_checksums(self, dest_dir=THIRDPARTY_DIR): @@ -826,8 +819,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): Fetch license files if missing in `dest_dir`. Return True if license files were fetched. """ - urls = LinksRepository.from_url( - use_cached_index=use_cached_index).links + urls = LinksRepository.from_url(use_cached_index=use_cached_index).links errors = [] extra_lic_names = [lic.get("file") for lic in self.extra_data.get("licenses", {})] extra_lic_names += [self.extra_data.get("license_file")] @@ -840,8 +832,7 @@ def fetch_license_files(self, dest_dir=THIRDPARTY_DIR, use_cached_index=False): try: # try remotely first - lic_url = get_license_link_for_filename( - filename=filename, urls=urls) + lic_url = get_license_link_for_filename(filename=filename, urls=urls) fetch_and_save( path_or_url=lic_url, @@ -919,8 +910,7 @@ def load_pkginfo_data(self, dest_dir=THIRDPARTY_DIR): c for c in classifiers if c.startswith("License") ] license_expression = get_license_expression(declared_license) - other_classifiers = [ - c for c in classifiers if not c.startswith("License")] + other_classifiers = [c for c in classifiers if not c.startswith("License")] holder = raw_data["Author"] holder_contact = raw_data["Author-email"] @@ -962,8 +952,7 @@ def update(self, data, overwrite=False, keep_extra=True): package_url = data.get("package_url") if package_url: purl_from_data = packageurl.PackageURL.from_string(package_url) - purl_from_self = packageurl.PackageURL.from_string( - self.package_url) + purl_from_self = packageurl.PackageURL.from_string(self.package_url) if purl_from_data != purl_from_self: print( f"Invalid dist update attempt, no same same purl with dist: " @@ -1013,8 +1002,7 @@ def get_license_link_for_filename(filename, urls): if not path_or_url: raise Exception(f"Missing link to file: {filename}") if not len(path_or_url) == 1: - raise Exception( - f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) + raise Exception(f"Multiple links to file: {filename}: \n" + "\n".join(path_or_url)) return path_or_url[0] @@ -1102,7 +1090,6 @@ def get_sdist_name_ver_ext(filename): @attr.attributes class Sdist(Distribution): - extension = attr.ib( repr=False, type=str, @@ -1407,8 +1394,7 @@ def packages_from_dir(cls, directory): """ base = os.path.abspath(directory) - paths = [os.path.join(base, f) - for f in os.listdir(base) if f.endswith(EXTENSIONS)] + paths = [os.path.join(base, f) for f in os.listdir(base) if f.endswith(EXTENSIONS)] if TRACE_ULTRA_DEEP: print("packages_from_dir: paths:", paths) @@ -1469,8 +1455,7 @@ def dists_from_paths_or_urls(cls, paths_or_urls): dists = [] if TRACE_ULTRA_DEEP: print(" ###paths_or_urls:", paths_or_urls) - installable = [f for f in paths_or_urls if f.endswith( - EXTENSIONS_INSTALLABLE)] + installable = [f for f in paths_or_urls if f.endswith(EXTENSIONS_INSTALLABLE)] for path_or_url in installable: try: dist = Distribution.from_path_or_url(path_or_url) @@ -1536,8 +1521,7 @@ class Environment: implementation = attr.ib( type=str, default="cp", - metadata=dict( - help="Python implementation supported by this environment."), + metadata=dict(help="Python implementation supported by this environment."), repr=False, ) @@ -1551,8 +1535,7 @@ class Environment: platforms = attr.ib( type=list, default=attr.Factory(list), - metadata=dict( - help="List of platform tags supported by this environment."), + metadata=dict(help="List of platform tags supported by this environment."), repr=False, ) @@ -1637,8 +1620,7 @@ class PypiSimpleRepository: fetched_package_normalized_names = attr.ib( type=set, default=attr.Factory(set), - metadata=dict( - help="A set of already fetched package normalized names."), + metadata=dict(help="A set of already fetched package normalized names."), ) use_cached_index = attr.ib( @@ -1671,12 +1653,10 @@ def _get_package_versions_map(self, name): self.packages[normalized_name] = versions except RemoteNotFetchedException as e: if TRACE: - print( - f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") + print(f"failed to fetch package name: {name} from: {self.index_url}:\n{e}") if not versions and TRACE: - print( - f"WARNING: package {name} not found in repo: {self.index_url}") + print(f"WARNING: package {name} not found in repo: {self.index_url}") return versions @@ -1861,8 +1841,7 @@ def get(self, path_or_url, as_text=True, force=False): if force or not os.path.exists(cached): if TRACE_DEEP: print(f" FILE CACHE MISS: {path_or_url}") - content = get_file_content( - path_or_url=path_or_url, as_text=as_text) + content = get_file_content(path_or_url=path_or_url, as_text=as_text) wmode = "w" if as_text else "wb" with open(cached, wmode) as fo: fo.write(content) @@ -1884,8 +1863,7 @@ def get_file_content(path_or_url, as_text=True): if path_or_url.startswith("https://"): if TRACE_DEEP: print(f"Fetching: {path_or_url}") - _headers, content = get_remote_file_content( - url=path_or_url, as_text=as_text) + _headers, content = get_remote_file_content(url=path_or_url, as_text=as_text) return content elif path_or_url.startswith("file://") or ( @@ -1936,7 +1914,7 @@ def get_remote_file_content( # several redirects and that we can ignore content there. A HEAD request may # not get us this last header print(f" DOWNLOADING: {url}") - with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113 + with requests.get(url, allow_redirects=True, stream=True, headers=headers) as response: # noqa: S113 status = response.status_code if status != requests.codes.ok: # NOQA if status == 429 and _delay < 20: @@ -1951,8 +1929,7 @@ def get_remote_file_content( ) else: - raise RemoteNotFetchedException( - f"Failed HTTP request from {url} with {status}") + raise RemoteNotFetchedException(f"Failed HTTP request from {url} with {status}") if headers_only: return response.headers, None @@ -2043,8 +2020,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # lets try to get from another dist of the same local package @@ -2056,8 +2032,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get another version of the same package that is not our version @@ -2068,8 +2043,7 @@ def get_other_dists(_package, _dist): ] other_local_version = other_local_packages and other_local_packages[-1] if other_local_version: - latest_local_dists = list( - other_local_version.get_distributions()) + latest_local_dists = list(other_local_version.get_distributions()) for latest_local_dist in latest_local_dists: latest_local_dist.load_about_data(dest_dir=dest_dir) if not latest_local_dist.has_key_metadata(): @@ -2095,8 +2069,7 @@ def get_other_dists(_package, _dist): # if has key data we may look to improve later, but we can move on if local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir=dest_dir) - local_dist.fetch_license_files( - dest_dir=dest_dir, use_cached_index=use_cached_index) + local_dist.fetch_license_files(dest_dir=dest_dir, use_cached_index=use_cached_index) continue # try to get a latest version of the same package that is not our version @@ -2137,8 +2110,7 @@ def get_other_dists(_package, _dist): # if local_dist.has_key_metadata() or not local_dist.has_key_metadata(): local_dist.save_about_and_notice_files(dest_dir) - lic_errs = local_dist.fetch_license_files( - dest_dir, use_cached_index=use_cached_index) + lic_errs = local_dist.fetch_license_files(dest_dir, use_cached_index=use_cached_index) if not local_dist.has_key_metadata(): print(f"Unable to add essential ABOUT data for: {local_dist}") @@ -2161,10 +2133,9 @@ def call(args, verbose=TRACE): """ if TRACE_DEEP: print("Calling:", " ".join(args)) - with subprocess.Popen( # noqa: S603 + with subprocess.Popen( # noqa: S603 args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8" ) as process: - stdouts = [] while True: line = process.stdout.readline() @@ -2260,7 +2231,7 @@ def download_wheels_with_pip( def check_about(dest_dir=THIRDPARTY_DIR): try: - subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603 + subprocess.check_output(f"venv/bin/about check {dest_dir}".split()) # noqa: S603 except subprocess.CalledProcessError as cpe: print() print("Invalid ABOUT files:") @@ -2286,8 +2257,7 @@ def find_problems( for dist in package.get_distributions(): dist.load_about_data(dest_dir=dest_dir) - abpth = os.path.abspath(os.path.join( - dest_dir, dist.about_filename)) + abpth = os.path.abspath(os.path.join(dest_dir, dist.about_filename)) if not dist.has_key_metadata(): print(f" Missing key ABOUT data in file://{abpth}") if "classifiers" in dist.extra_data: From 1189dda52570ed018f52eacd38c4990e8be8ff1a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:02:19 +0100 Subject: [PATCH 066/120] Apply cosmetic refactorings Signed-off-by: Philippe Ombredanne --- docs/source/conf.py | 3 ++- etc/scripts/check_thirdparty.py | 4 +--- etc/scripts/fetch_thirdparty.py | 17 ++++++++--------- etc/scripts/gen_pypi_simple.py | 15 +++++++-------- etc/scripts/gen_requirements.py | 4 ++-- etc/scripts/gen_requirements_dev.py | 4 ++-- .../test_utils_pip_compatibility_tags.py | 9 +++++---- etc/scripts/utils_dejacode.py | 9 +++++---- etc/scripts/utils_pip_compatibility_tags.py | 8 +++++--- etc/scripts/utils_requirements.py | 3 +-- etc/scripts/utils_thirdparty.py | 3 ++- 11 files changed, 40 insertions(+), 39 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 8c88fa2c..8aad8294 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -94,7 +94,8 @@ html_show_sphinx = True # Define CSS and HTML abbreviations used in .rst files. These are examples. -# .. role:: is used to refer to styles defined in _static/theme_overrides.css and is used like this: :red:`text` +# .. role:: is used to refer to styles defined in _static/theme_overrides.css +# and is used like this: :red:`text` rst_prolog = """ .. |psf| replace:: Python Software Foundation diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 1aa4e28a..bb8347a5 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -41,8 +40,7 @@ def check_thirdparty_dir( """ Check a thirdparty directory for problems and print these on screen. """ - # check for problems - print(f"==> CHECK FOR PROBLEMS") + print("==> CHECK FOR PROBLEMS") utils_thirdparty.find_problems( dest_dir=dest, report_missing_sources=sdists, diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py index c2246837..76a19a60 100644 --- a/etc/scripts/fetch_thirdparty.py +++ b/etc/scripts/fetch_thirdparty.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -10,7 +9,6 @@ # import itertools -import os import sys from collections import defaultdict @@ -109,7 +107,8 @@ @click.option( "--use-cached-index", is_flag=True, - help="Use on disk cached PyPI indexes list of packages and versions and do not refetch if present.", + help="Use on disk cached PyPI indexes list of packages and versions and " + "do not refetch if present.", ) @click.option( "--sdist-only", @@ -261,7 +260,7 @@ def fetch_thirdparty( if not fetched: wheels_or_sdist_not_found[f"{name}=={version}"].append(environment) if TRACE: - print(f" NOT FOUND") + print(" NOT FOUND") if sdists or (f"{name}=={version}" in wheels_or_sdist_not_found and name in sdist_only): if TRACE: @@ -276,17 +275,17 @@ def fetch_thirdparty( if not fetched: wheels_or_sdist_not_found[f"{name}=={version}"].append("sdist") if TRACE: - print(f" NOT FOUND") + print(" NOT FOUND") mia = [] for nv, dists in wheels_or_sdist_not_found.items(): name, _, version = nv.partition("==") if name in no_dist: continue - sdist_missing = sdists and "sdist" in dists and not name in wheel_only + sdist_missing = sdists and "sdist" in dists and name not in wheel_only if sdist_missing: mia.append(f"SDist missing: {nv} {dists}") - wheels_missing = wheels and any(d for d in dists if d != "sdist") and not name in sdist_only + wheels_missing = wheels and any(d for d in dists if d != "sdist") and name not in sdist_only if wheels_missing: mia.append(f"Wheels missing: {nv} {dists}") @@ -295,12 +294,12 @@ def fetch_thirdparty( print(m) raise Exception(mia) - print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") + print("==> FETCHING OR CREATING ABOUT AND LICENSE FILES") utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir, use_cached_index=use_cached_index) utils_thirdparty.clean_about_files(dest_dir=dest_dir) # check for problems - print(f"==> CHECK FOR PROBLEMS") + print("==> CHECK FOR PROBLEMS") utils_thirdparty.find_problems( dest_dir=dest_dir, report_missing_sources=sdists, diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index cfe68e67..89d06265 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # SPDX-License-Identifier: BSD-2-Clause-Views AND MIT # Copyright (c) 2010 David Wolever . All rights reserved. @@ -132,7 +131,7 @@ def build_links_package_index(packages_by_package_name, base_url): Return an HTML document as string which is a links index of all packages """ document = [] - header = f""" + header = """ Links for all packages @@ -177,13 +176,13 @@ def simple_index_entry(self, base_url): def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi"): """ - Using a ``directory`` directory of wheels and sdists, create the a PyPI - simple directory index at ``directory``/simple/ populated with the proper - PyPI simple index directory structure crafted using symlinks. + Create the a PyPI simple directory index using a ``directory`` directory of wheels and sdists in + the direvctory at ``directory``/simple/ populated with the proper PyPI simple index directory + structure crafted using symlinks. - WARNING: The ``directory``/simple/ directory is removed if it exists. - NOTE: in addition to the a PyPI simple index.html there is also a links.html - index file generated which is suitable to use with pip's --find-links + WARNING: The ``directory``/simple/ directory is removed if it exists. NOTE: in addition to the a + PyPI simple index.html there is also a links.html index file generated which is suitable to use + with pip's --find-links """ directory = Path(directory) diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py index 2b65ae80..1b879442 100644 --- a/etc/scripts/gen_requirements.py +++ b/etc/scripts/gen_requirements.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -34,7 +33,8 @@ def gen_requirements(): type=pathlib.Path, required=True, metavar="DIR", - help="Path to the 'site-packages' directory where wheels are installed such as lib/python3.6/site-packages", + help="Path to the 'site-packages' directory where wheels are installed " + "such as lib/python3.12/site-packages", ) parser.add_argument( "-r", diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py index 5db1c48e..85482056 100644 --- a/etc/scripts/gen_requirements_dev.py +++ b/etc/scripts/gen_requirements_dev.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -36,7 +35,8 @@ def gen_dev_requirements(): type=pathlib.Path, required=True, metavar="DIR", - help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', + help="Path to the 'site-packages' directory where wheels are installed " + "such as lib/python3.12/site-packages", ) parser.add_argument( "-d", diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py index a33b8b38..de4b7066 100644 --- a/etc/scripts/test_utils_pip_compatibility_tags.py +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -1,4 +1,5 @@ -"""Generate and work with PEP 425 Compatibility Tags. +""" +Generate and work with PEP 425 Compatibility Tags. copied from pip-20.3.1 pip/tests/unit/test_utils_compatibility_tags.py download_url: https://raw.githubusercontent.com/pypa/pip/20.3.1/tests/unit/test_utils_compatibility_tags.py @@ -50,7 +51,7 @@ def test_version_info_to_nodot(version_info, expected): assert actual == expected -class Testcompatibility_tags(object): +class Testcompatibility_tags: def mock_get_config_var(self, **kwd): """ Patch sysconfig.get_config_var for arbitrary keys. @@ -81,7 +82,7 @@ def test_no_hyphen_tag(self): assert "-" not in tag.platform -class TestManylinux2010Tags(object): +class TestManylinux2010Tags: @pytest.mark.parametrize( "manylinux2010,manylinux1", [ @@ -104,7 +105,7 @@ def test_manylinux2010_implies_manylinux1(self, manylinux2010, manylinux1): assert arches[:2] == [manylinux2010, manylinux1] -class TestManylinux2014Tags(object): +class TestManylinux2014Tags: @pytest.mark.parametrize( "manylinuxA,manylinuxB", [ diff --git a/etc/scripts/utils_dejacode.py b/etc/scripts/utils_dejacode.py index cd39cda3..b6bff518 100644 --- a/etc/scripts/utils_dejacode.py +++ b/etc/scripts/utils_dejacode.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -25,7 +24,7 @@ DEJACODE_API_URL_PACKAGES = f"{DEJACODE_API_URL}packages/" DEJACODE_API_HEADERS = { - "Authorization": "Token {}".format(DEJACODE_API_KEY), + "Authorization": f"Token {DEJACODE_API_KEY}", "Accept": "application/json; indent=4", } @@ -50,6 +49,7 @@ def fetch_dejacode_packages(params): DEJACODE_API_URL_PACKAGES, params=params, headers=DEJACODE_API_HEADERS, + timeout=10, ) return response.json()["results"] @@ -93,7 +93,7 @@ def update_with_dejacode_about_data(distribution): if package_data: package_api_url = package_data["api_url"] about_url = f"{package_api_url}about" - response = requests.get(about_url, headers=DEJACODE_API_HEADERS) + response = requests.get(about_url, headers=DEJACODE_API_HEADERS, timeout=10) # note that this is YAML-formatted about_text = response.json()["about_data"] about_data = saneyaml.load(about_text) @@ -113,7 +113,7 @@ def fetch_and_save_about_files(distribution, dest_dir="thirdparty"): if package_data: package_api_url = package_data["api_url"] about_url = f"{package_api_url}about_files" - response = requests.get(about_url, headers=DEJACODE_API_HEADERS) + response = requests.get(about_url, headers=DEJACODE_API_HEADERS, timeout=10) about_zip = response.content with io.BytesIO(about_zip) as zf: with zipfile.ZipFile(zf) as zi: @@ -201,6 +201,7 @@ def create_dejacode_package(distribution): DEJACODE_API_URL_PACKAGES, data=new_package_payload, headers=DEJACODE_API_HEADERS, + timeout=10, ) new_package_data = response.json() if response.status_code != 201: diff --git a/etc/scripts/utils_pip_compatibility_tags.py b/etc/scripts/utils_pip_compatibility_tags.py index de0ac95d..dd954bca 100644 --- a/etc/scripts/utils_pip_compatibility_tags.py +++ b/etc/scripts/utils_pip_compatibility_tags.py @@ -1,4 +1,5 @@ -"""Generate and work with PEP 425 Compatibility Tags. +""" +Generate and work with PEP 425 Compatibility Tags. copied from pip-20.3.1 pip/_internal/utils/compatibility_tags.py download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/utils/compatibility_tags.py @@ -130,7 +131,7 @@ def _get_custom_interpreter(implementation=None, version=None): implementation = interpreter_name() if version is None: version = interpreter_version() - return "{}{}".format(implementation, version) + return f"{implementation}{version}" def get_supported( @@ -140,7 +141,8 @@ def get_supported( abis=None, # type: Optional[List[str]] ): # type: (...) -> List[Tag] - """Return a list of supported tags for each version specified in + """ + Return a list of supported tags for each version specified in `versions`. :param version: a string version, of the form "33" or "32", diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index 167bc9f5..b9b2c0e7 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- # # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. @@ -40,7 +39,7 @@ def get_required_name_versions(requirement_lines, with_unpinned=False): req_line = req_line.strip() if not req_line or req_line.startswith("#"): continue - if req_line.startswith("-") or (not with_unpinned and not "==" in req_line): + if req_line.startswith("-") or (not with_unpinned and "==" not in req_line): print(f"Requirement line is not supported: ignored: {req_line}") continue yield get_required_name_version(requirement=req_line, with_unpinned=with_unpinned) diff --git a/etc/scripts/utils_thirdparty.py b/etc/scripts/utils_thirdparty.py index 4ea1babb..aafc1d69 100644 --- a/etc/scripts/utils_thirdparty.py +++ b/etc/scripts/utils_thirdparty.py @@ -91,7 +91,8 @@ - parse requirement file - create a TODO queue of requirements to process -- done: create an empty map of processed binary requirements as {package name: (list of versions/tags} +- done: create an empty map of processed binary requirements as + {package name: (list of versions/tags} - while we have package reqs in TODO queue, process one requirement: From 84257fbe200e5780cb13536a5c8eb56a88539e6a Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:05:23 +0100 Subject: [PATCH 067/120] Reformat test code Signed-off-by: Philippe Ombredanne --- .gitignore | 1 + pyproject.toml | 19 +++++++++++-------- tests/test_skeleton_codestyle.py | 25 ++++++++++++++++--------- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 2d48196f..8a93c94d 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,4 @@ tcl # Ignore Jupyter Notebook related temp files .ipynb_checkpoints/ +/.ruff_cache/ diff --git a/pyproject.toml b/pyproject.toml index ba55770f..a872ab3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,16 +67,17 @@ include = [ [tool.ruff.lint] # Rules: https://docs.astral.sh/ruff/rules/ select = [ - "E", # pycodestyle - "W", # pycodestyle warnings - "D", # pydocstyle - "F", # Pyflakes - "UP", # pyupgrade - "S", # flake8-bandit +# "E", # pycodestyle +# "W", # pycodestyle warnings +# "D", # pydocstyle +# "F", # Pyflakes +# "UP", # pyupgrade +# "S", # flake8-bandit "I", # isort - "C9", # McCabe complexity +# "C9", # McCabe complexity ] -ignore = ["D1", "D200", "D203", "D205", "D212", "D400", "D415"] +ignore = ["D1", "D200", "D202", "D203", "D205", "D212", "D400", "D415"] + [tool.ruff.lint.isort] force-single-line = true @@ -100,3 +101,5 @@ max-complexity = 10 [tool.ruff.lint.per-file-ignores] # Place paths of files to be ignored by ruff here +"tests/*" = ["S101"] +"test_*.py" = ["S101"] diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py index b4ce8c16..8cd85c94 100644 --- a/tests/test_skeleton_codestyle.py +++ b/tests/test_skeleton_codestyle.py @@ -7,30 +7,37 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import configparser import subprocess import unittest -import configparser - class BaseTests(unittest.TestCase): def test_skeleton_codestyle(self): - """ - This test shouldn't run in proliferated repositories. - """ + # This test shouldn't run in proliferated repositories. + + # TODO: update with switch to pyproject.toml setup_cfg = configparser.ConfigParser() setup_cfg.read("setup.cfg") if setup_cfg["metadata"]["name"] != "skeleton": return - args = "venv/bin/black --check -l 100 setup.py etc tests" + commands = [ + ["venv/bin/ruff", "--check"], + ["venv/bin/ruff", "format", "--check"], + ] + command = None try: - subprocess.check_output(args.split()) + for command in commands: + subprocess.check_output(command) # noqa: S603 except subprocess.CalledProcessError as e: print("===========================================================") print(e.output) print("===========================================================") raise Exception( - "Black style check failed; please format the code using:\n" - " python -m black -l 100 setup.py etc tests", + f"Code style and linting command check failed: {' '.join(command)!r}.\n" + "You can check and format the code using:\n" + " make valid\n", + "OR:\n ruff format\n", + " ruff check --fix\n", e.output, ) from e From 00684f733a0873bd837af85471814907ba93f456 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:08:25 +0100 Subject: [PATCH 068/120] Format code Signed-off-by: Philippe Ombredanne --- etc/scripts/check_thirdparty.py | 1 + etc/scripts/test_utils_pip_compatibility_tags.py | 1 + tests/test_skeleton_codestyle.py | 1 + 3 files changed, 3 insertions(+) diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index bb8347a5..65ae595e 100644 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -11,6 +11,7 @@ import utils_thirdparty + @click.command() @click.option( "-d", diff --git a/etc/scripts/test_utils_pip_compatibility_tags.py b/etc/scripts/test_utils_pip_compatibility_tags.py index de4b7066..0e9c360a 100644 --- a/etc/scripts/test_utils_pip_compatibility_tags.py +++ b/etc/scripts/test_utils_pip_compatibility_tags.py @@ -33,6 +33,7 @@ import utils_pip_compatibility_tags + @pytest.mark.parametrize( "version_info, expected", [ diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py index 8cd85c94..7135ac0d 100644 --- a/tests/test_skeleton_codestyle.py +++ b/tests/test_skeleton_codestyle.py @@ -11,6 +11,7 @@ import subprocess import unittest + class BaseTests(unittest.TestCase): def test_skeleton_codestyle(self): # This test shouldn't run in proliferated repositories. From 7c4278df4e8acf04888a188d115b4c687060f1e5 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:10:45 +0100 Subject: [PATCH 069/120] Refine ruff configuration Signed-off-by: Philippe Ombredanne --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a872ab3a..0f8bd587 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,11 +72,11 @@ select = [ # "D", # pydocstyle # "F", # Pyflakes # "UP", # pyupgrade -# "S", # flake8-bandit + "S", # flake8-bandit "I", # isort # "C9", # McCabe complexity ] -ignore = ["D1", "D200", "D202", "D203", "D205", "D212", "D400", "D415"] +ignore = ["D1", "D200", "D202", "D203", "D205", "D212", "D400", "D415", "I001"] [tool.ruff.lint.isort] From 47cb840db13d3e7328dba8d8e62197cda82e48ec Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:54:01 +0100 Subject: [PATCH 070/120] Format doc Signed-off-by: Philippe Ombredanne --- AUTHORS.rst | 2 +- README.rst | 29 ++++++++++++++--------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/AUTHORS.rst b/AUTHORS.rst index 51a19cc8..16e20464 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -1,3 +1,3 @@ The following organizations or individuals have contributed to this repo: -- +- diff --git a/README.rst b/README.rst index f848b4b3..01d02108 100644 --- a/README.rst +++ b/README.rst @@ -1,11 +1,10 @@ A Simple Python Project Skeleton ================================ -This repo attempts to standardize the structure of the Python-based project's repositories using -modern Python packaging and configuration techniques that can then be applied to many repos. - -Using this `blog post`_ as inspiration, this repository serves as the base for all new Python -projects and is mergeable in existing repositories as well. +This repo attempts to standardize the structure of the Python-based project's +repositories using modern Python packaging and configuration techniques. +Using this `blog post`_ as inspiration, this repository serves as the base for +all new Python projects and is mergeable in existing repositories as well. .. _blog post: https://blog.jaraco.com/a-project-skeleton-for-python-projects/ @@ -69,7 +68,7 @@ Release Notes - Drop support for Python 3.8 - Drop support for macOS-11, add support for macOS-14 - + - 2024-02-19: - Replace support in CI of default ubuntu-20.04 by ubuntu-22.04 @@ -86,19 +85,19 @@ Release Notes - Synchronize configure and configure.bat scripts for sanity - Update CI operating system support with latest Azure OS images - - Streamline utility scripts in etc/scripts/ to create, fetch and manage third-party dependencies - There are now fewer scripts. See etc/scripts/README.rst for details + - Streamline utility scripts in etc/scripts/ to create, fetch and manage third-party + dependencies. There are now fewer scripts. See etc/scripts/README.rst for details - 2021-09-03: - - - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` and ``requirements-dev.txt`` - + - ``configure`` now requires pinned dependencies via the use of ``requirements.txt`` + and ``requirements-dev.txt`` - ``configure`` can now accept multiple options at once - Add utility scripts from scancode-toolkit/etc/release/ for use in generating project files - Rename virtual environment directory from ``tmp`` to ``venv`` - - Update README.rst with instructions for generating ``requirements.txt`` and ``requirements-dev.txt``, - as well as collecting dependencies as wheels and generating ABOUT files for them. + - Update README.rst with instructions for generating ``requirements.txt`` + and ``requirements-dev.txt``, as well as collecting dependencies as wheels and generating + ABOUT files for them. - 2021-05-11: - - - Adopt new configure scripts from ScanCode TK that allows correct configuration of which Python version is used. + - Adopt new configure scripts from ScanCode TK that allows correct configuration of which + Python version is used. From 7b29b5914a443069a7ff967eb4ef096034333248 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:54:35 +0100 Subject: [PATCH 071/120] Run doc8 on all rst files Signed-off-by: Philippe Ombredanne --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 930e8010..debc404e 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ dev: doc8: @echo "-> Run doc8 validation" - @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ + @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ *.rst valid: @echo "-> Run Ruff format" From 86c7ca45d3132e5f6873658ab1743b6e27cfeb58 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sat, 29 Mar 2025 22:55:20 +0100 Subject: [PATCH 072/120] Enable doc style checks Signed-off-by: Philippe Ombredanne --- pyproject.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0f8bd587..51761ff8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,8 @@ include = [ "etc/**/*.py", "test/**/*.py", "doc/**/*", - "*.py" + "*.py", + "." ] [tool.ruff.lint] @@ -69,10 +70,10 @@ include = [ select = [ # "E", # pycodestyle # "W", # pycodestyle warnings -# "D", # pydocstyle + "D", # pydocstyle # "F", # Pyflakes # "UP", # pyupgrade - "S", # flake8-bandit +# "S", # flake8-bandit "I", # isort # "C9", # McCabe complexity ] From 71583c5ecdfe5dd352b7f2bb9d26deaad971e151 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 14:40:36 +0200 Subject: [PATCH 073/120] Do not format more test data Signed-off-by: Philippe Ombredanne --- pyproject.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 51761ff8..7d807ebf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,10 +60,25 @@ include = [ "src/**/*.py", "etc/**/*.py", "test/**/*.py", + "tests/**/*.py", "doc/**/*", + "docs/**/*", "*.py", "." ] +# ignore test data and testfiles: they should never be linted nor formatted +exclude = [ +# main style + "**/tests/data/**/*", +# scancode-toolkit + "**/tests/*/data/**/*", +# dejacode, purldb + "**/tests/testfiles/**/*", +# vulnerablecode, fetchcode + "**/tests/*/test_data/**/*", + "**/tests/test_data/**/*", +] + [tool.ruff.lint] # Rules: https://docs.astral.sh/ruff/rules/ From 0f1a40382bdcadf82512395787faab50927256f6 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 14:58:36 +0200 Subject: [PATCH 074/120] Do not treat rst as Python Signed-off-by: Philippe Ombredanne --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7d807ebf..5e16b564 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,8 +61,8 @@ include = [ "etc/**/*.py", "test/**/*.py", "tests/**/*.py", - "doc/**/*", - "docs/**/*", + "doc/**/*.py", + "docs/**/*.py", "*.py", "." ] From 6bea6577864bf432438dabaed9e46a721aae2961 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 16:41:57 +0200 Subject: [PATCH 075/120] Combine testing and docs extra for simplicity Signed-off-by: Philippe Ombredanne --- configure | 2 -- configure.bat | 4 ---- setup.cfg | 3 --- 3 files changed, 9 deletions(-) diff --git a/configure b/configure index 22d92885..83fd2035 100755 --- a/configure +++ b/configure @@ -30,7 +30,6 @@ CLI_ARGS=$1 # Requirement arguments passed to pip and used by default or with --dev. REQUIREMENTS="--editable . --constraint requirements.txt" DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" -DOCS_REQUIREMENTS="--editable .[docs] --constraint requirements.txt" # where we create a virtualenv VIRTUALENV_DIR=venv @@ -185,7 +184,6 @@ while getopts :-: optchar; do help ) cli_help;; clean ) find_python && clean;; dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; - docs ) CFG_REQUIREMENTS="$DOCS_REQUIREMENTS";; esac;; esac done diff --git a/configure.bat b/configure.bat index 5b9a9d68..18b37038 100644 --- a/configure.bat +++ b/configure.bat @@ -28,7 +28,6 @@ @rem # Requirement arguments passed to pip and used by default or with --dev. set "REQUIREMENTS=--editable . --constraint requirements.txt" set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" -set "DOCS_REQUIREMENTS=--editable .[docs] --constraint requirements.txt" @rem # where we create a virtualenv set "VIRTUALENV_DIR=venv" @@ -76,9 +75,6 @@ if not "%1" == "" ( if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" ) - if "%1" EQU "--docs" ( - set "CFG_REQUIREMENTS=%DOCS_REQUIREMENTS%" - ) shift goto again ) diff --git a/setup.cfg b/setup.cfg index aaec643f..ad8e0d8f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,8 +55,6 @@ testing = pycodestyle >= 2.8.0 twine ruff - -docs = Sphinx>=5.0.2 sphinx-rtd-theme>=1.0.0 sphinx-reredirects >= 0.1.2 @@ -64,4 +62,3 @@ docs = sphinx-autobuild sphinx-rtd-dark-mode>=1.3.0 sphinx-copybutton - From c615589b54bfac74b6f17b2233b2afe60bf1d0f6 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 17:18:19 +0200 Subject: [PATCH 076/120] Refine checking of docs with doc8 Signed-off-by: Philippe Ombredanne --- Makefile | 2 +- pyproject.toml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index debc404e..d21a2f95 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ dev: doc8: @echo "-> Run doc8 validation" - @${ACTIVATE} doc8 --max-line-length 100 --ignore-path docs/_build/ --quiet docs/ *.rst + @${ACTIVATE} doc8 docs/ *.rst valid: @echo "-> Run Ruff format" diff --git a/pyproject.toml b/pyproject.toml index 5e16b564..bfb1d353 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,3 +119,10 @@ max-complexity = 10 # Place paths of files to be ignored by ruff here "tests/*" = ["S101"] "test_*.py" = ["S101"] + + +[tool.doc8] + +ignore-path = ["docs/build", "doc/build", "docs/_build", "doc/_build"] +max-line-length=100 +verbose=0 From 04e0a89a3bbf5359f27b6e3ef9a5026638e63de8 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 18:41:00 +0200 Subject: [PATCH 077/120] Refine doc handling * remove CI scripts and use Makefile targets instead * ensure doc8 runs quiet * add new docs-check make target to run documentation and links checks * update oudated doc for docs contribution Signed-off-by: Philippe Ombredanne --- .github/workflows/docs-ci.yml | 12 +++++------- Makefile | 10 +++++++--- docs/scripts/doc8_style_check.sh | 5 ----- docs/scripts/sphinx_build_link_check.sh | 5 ----- docs/source/conf.py | 2 +- docs/source/contribute/contrib_doc.rst | 8 ++++---- pyproject.toml | 2 -- 7 files changed, 17 insertions(+), 27 deletions(-) delete mode 100755 docs/scripts/doc8_style_check.sh delete mode 100644 docs/scripts/sphinx_build_link_check.sh diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 621de4b2..10ba5faa 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -21,14 +21,12 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Dependencies - run: pip install -e .[docs] + run: ./configure --dev - - name: Check Sphinx Documentation build minimally - working-directory: ./docs - run: sphinx-build -E -W source build + - name: Check documentation and HTML for errors and dead links + run: make docs-check - - name: Check for documentation style errors - working-directory: ./docs - run: ./scripts/doc8_style_check.sh + - name: Check documentation for style errors + run: make doc8 diff --git a/Makefile b/Makefile index d21a2f95..413399e5 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ dev: doc8: @echo "-> Run doc8 validation" - @${ACTIVATE} doc8 docs/ *.rst + @${ACTIVATE} doc8 --quiet docs/ *.rst valid: @echo "-> Run Ruff format" @@ -46,6 +46,10 @@ test: docs: rm -rf docs/_build/ - @${ACTIVATE} sphinx-build docs/ docs/_build/ + @${ACTIVATE} sphinx-build docs/source docs/_build/ -.PHONY: conf dev check valid clean test docs +docs-check: + @${ACTIVATE} sphinx-build -E -W -b html docs/source docs/_build/ + @${ACTIVATE} sphinx-build -E -W -b linkcheck docs/source docs/_build/ + +.PHONY: conf dev check valid clean test docs docs-check diff --git a/docs/scripts/doc8_style_check.sh b/docs/scripts/doc8_style_check.sh deleted file mode 100755 index 94163239..00000000 --- a/docs/scripts/doc8_style_check.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# halt script on error -set -e -# Check for Style Code Violations -doc8 --max-line-length 100 source --ignore D000 --quiet \ No newline at end of file diff --git a/docs/scripts/sphinx_build_link_check.sh b/docs/scripts/sphinx_build_link_check.sh deleted file mode 100644 index c5426863..00000000 --- a/docs/scripts/sphinx_build_link_check.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -# halt script on error -set -e -# Build locally, and then check links -sphinx-build -E -W -b linkcheck source build \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 8aad8294..056ca6ea 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -18,7 +18,7 @@ # -- Project information ----------------------------------------------------- project = "nexb-skeleton" -copyright = "nexB Inc. and others." +copyright = "nexB Inc., AboutCode and others." author = "AboutCode.org authors and contributors" diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst index 5640db26..041b3583 100644 --- a/docs/source/contribute/contrib_doc.rst +++ b/docs/source/contribute/contrib_doc.rst @@ -147,7 +147,7 @@ What is Checked? ^^^^^^^^^^^^^^^^ PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. -Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. +Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. What is checked: @@ -169,11 +169,11 @@ What is checked: Interspinx ---------- -ScanCode toolkit documentation uses `Intersphinx `_ +ScanCode toolkit documentation uses `Intersphinx `_ to link to other Sphinx Documentations, to maintain links to other Aboutcode Projects. To link sections in the same documentation, standart reST labels are used. Refer -`Cross-Referencing `_ for more information. +`Cross-Referencing `_ for more information. For example:: @@ -230,7 +230,7 @@ Style Conventions for the Documentaion 1. Headings - (`Refer `_) + (`Refer `_) Normally, there are no heading levels assigned to certain characters as the structure is determined from the succession of headings. However, this convention is used in Python’s Style Guide for documenting which you may follow: diff --git a/pyproject.toml b/pyproject.toml index bfb1d353..c9e67720 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -122,7 +122,5 @@ max-complexity = 10 [tool.doc8] - ignore-path = ["docs/build", "doc/build", "docs/_build", "doc/_build"] max-line-length=100 -verbose=0 From 8897cc63eb9ef9b06a1fdc77ebfe21289c69961b Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 18:49:01 +0200 Subject: [PATCH 078/120] Add twine check to release publication Signed-off-by: Philippe Ombredanne --- .github/workflows/pypi-release.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index a66c9c80..cf0579a7 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -30,12 +30,15 @@ jobs: with: python-version: 3.12 - - name: Install pypa/build - run: python -m pip install build --user + - name: Install pypa/build and twine + run: python -m pip install --user build twine - name: Build a binary wheel and a source tarball run: python -m build --sdist --wheel --outdir dist/ + - name: Validate wheel and sdis for Pypi + run: python -m twine check dist/* + - name: Upload built archives uses: actions/upload-artifact@v4 with: From 3d42985990860188c5fbf9f64f3fd4d14c590a65 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Sun, 30 Mar 2025 19:16:31 +0200 Subject: [PATCH 079/120] Refine doc contribution docs Signed-off-by: Philippe Ombredanne --- docs/source/contribute/contrib_doc.rst | 119 ++++++++----------------- 1 file changed, 38 insertions(+), 81 deletions(-) diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst index 041b3583..dee9296d 100644 --- a/docs/source/contribute/contrib_doc.rst +++ b/docs/source/contribute/contrib_doc.rst @@ -8,109 +8,59 @@ Contributing to the Documentation Setup Local Build ----------------- -To get started, create or identify a working directory on your local machine. +To get started, check out and configure the repository for development:: -Open that directory and execute the following command in a terminal session:: + git clone https://github.com/aboutcode-org/.git - git clone https://github.com/aboutcode-org/skeleton.git + cd your-repo + ./configure --dev -That will create an ``/skeleton`` directory in your working directory. -Now you can install the dependencies in a virtualenv:: - - cd skeleton - ./configure --docs +(Or use "make dev") .. note:: - In case of windows, run ``configure --docs`` instead of this. - -Now, this will install the following prerequisites: - -- Sphinx -- sphinx_rtd_theme (the format theme used by ReadTheDocs) -- docs8 (style linter) + In case of windows, run ``configure --dev``. -These requirements are already present in setup.cfg and `./configure --docs` installs them. +This will install and configure all requirements foer development including for docs development. -Now you can build the HTML documents locally:: +Now you can build the HTML documentation locally:: source venv/bin/activate - cd docs - make html - -Assuming that your Sphinx installation was successful, Sphinx should build a local instance of the -documentation .html files:: - - open build/html/index.html - -.. note:: - - In case this command did not work, for example on Ubuntu 18.04 you may get a message like “Couldn’t - get a file descriptor referring to the console”, try: - - :: - - see build/html/index.html + make docs -You now have a local build of the AboutCode documents. +This will build a local instance of the ``docs/_build`` directory:: -.. _contrib_doc_share_improvements: + open docs/_build/index.html -Share Document Improvements ---------------------------- - -Ensure that you have the latest files:: - - git pull - git status -Before commiting changes run Continious Integration Scripts locally to run tests. Refer -:ref:`doc_ci` for instructions on the same. +To validate the documentation style and content, use:: -Follow standard git procedures to upload your new and modified files. The following commands are -examples:: - - git status - git add source/index.rst - git add source/how-to-scan.rst - git status - git commit -m "New how-to document that explains how to scan" - git status - git push - git status - -The Scancode-Toolkit webhook with ReadTheDocs should rebuild the documentation after your -Pull Request is Merged. + source venv/bin/activate + make doc8 + make docs-check -Refer the `Pro Git Book `_ available online for Git tutorials -covering more complex topics on Branching, Merging, Rebasing etc. .. _doc_ci: Continuous Integration ---------------------- -The documentations are checked on every new commit through Travis-CI, so that common errors are -avoided and documentation standards are enforced. Travis-CI presently checks for these 3 aspects -of the documentation : +The documentations are checked on every new commit, so that common errors are avoided and +documentation standards are enforced. We checks for these aspects of the documentation: 1. Successful Builds (By using ``sphinx-build``) -2. No Broken Links (By Using ``link-check``) -3. Linting Errors (By Using ``Doc8``) +2. No Broken Links (By Using ``linkcheck``) +3. Linting Errors (By Using ``doc8``) -So run these scripts at your local system before creating a Pull Request:: +You myst run these scripts locally before creating a pull request:: - cd docs - ./scripts/sphinx_build_link_check.sh - ./scripts/doc8_style_check.sh + make doc8 + make check-docs -If you don't have permission to run the scripts, run:: - - chmod u+x ./scripts/doc8_style_check.sh .. _doc_style_docs8: -Style Checks Using ``Doc8`` +Style Checks Using ``doc8`` --------------------------- How To Run Style Tests @@ -118,8 +68,7 @@ How To Run Style Tests In the project root, run the following commands:: - $ cd docs - $ ./scripts/doc8_style_check.sh + make doc8 A sample output is:: @@ -143,11 +92,13 @@ A sample output is:: Now fix the errors and run again till there isn't any style error in the documentation. + What is Checked? ^^^^^^^^^^^^^^^^ PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. -Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. +Doc8 is a sub-project of the same Organization. Refer this +`README `_ for more details. What is checked: @@ -164,16 +115,19 @@ What is checked: - no carriage returns (use UNIX newlines) - D004 - no newline at end of file - D005 + .. _doc_interspinx: Interspinx ---------- -ScanCode toolkit documentation uses `Intersphinx `_ +AboutCode documentation uses +`Intersphinx `_ to link to other Sphinx Documentations, to maintain links to other Aboutcode Projects. To link sections in the same documentation, standart reST labels are used. Refer -`Cross-Referencing `_ for more information. +`Cross-Referencing `_ +for more information. For example:: @@ -223,6 +177,7 @@ Intersphinx, and you link to that label, it will create a link to the local labe For more information, refer this tutorial named `Using Intersphinx `_. + .. _doc_style_conv: Style Conventions for the Documentaion @@ -303,12 +258,14 @@ Style Conventions for the Documentaion ``rst_snippets/warning_snippets/`` and then included to eliminate redundancy, as these are frequently used in multiple files. + Converting from Markdown ------------------------ -If you want to convert a ``.md`` file to a ``.rst`` file, this `tool `_ -does it pretty well. You'd still have to clean up and check for errors as this contains a lot of -bugs. But this is definitely better than converting everything by yourself. +If you want to convert a ``.md`` file to a ``.rst`` file, this +`tool `_ does it pretty well. +You will still have to clean up and check for errors as this contains a lot of bugs. But this is +definitely better than converting everything by yourself. This will be helpful in converting GitHub wiki's (Markdown Files) to reStructuredtext files for Sphinx/ReadTheDocs hosting. From f428366859a143add93160bd0b1ae685be89fcbb Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 31 Mar 2025 13:35:36 -0700 Subject: [PATCH 080/120] Update codestyle command * Remove trailing whitespace Signed-off-by: Jono Yang --- docs/source/contribute/contrib_doc.rst | 4 ++-- tests/test_skeleton_codestyle.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst index dee9296d..2a719a52 100644 --- a/docs/source/contribute/contrib_doc.rst +++ b/docs/source/contribute/contrib_doc.rst @@ -97,7 +97,7 @@ What is Checked? ^^^^^^^^^^^^^^^^ PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. -Doc8 is a sub-project of the same Organization. Refer this +Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. What is checked: @@ -263,7 +263,7 @@ Converting from Markdown ------------------------ If you want to convert a ``.md`` file to a ``.rst`` file, this -`tool `_ does it pretty well. +`tool `_ does it pretty well. You will still have to clean up and check for errors as this contains a lot of bugs. But this is definitely better than converting everything by yourself. diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py index 7135ac0d..6060c085 100644 --- a/tests/test_skeleton_codestyle.py +++ b/tests/test_skeleton_codestyle.py @@ -23,7 +23,7 @@ def test_skeleton_codestyle(self): return commands = [ - ["venv/bin/ruff", "--check"], + ["venv/bin/ruff", "check"], ["venv/bin/ruff", "format", "--check"], ] command = None From f0d0e21d5e6f98645b02ff9a8fee6ee3def1be75 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 31 Mar 2025 13:44:30 -0700 Subject: [PATCH 081/120] Update README.rst Signed-off-by: Jono Yang --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 01d02108..11a4dfb0 100644 --- a/README.rst +++ b/README.rst @@ -44,6 +44,10 @@ More usage instructions can be found in ``docs/skeleton-usage.rst``. Release Notes ============= +- 2025-03-31: + + - Use ruff as the main code formatting tool, add ruff rules to pyproject.toml + - 2025-03-29: - Add support for beta macOS-15 From f3a8aa6cee5f645a668750ab7e6bf0cdc774e041 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Mon, 31 Mar 2025 14:31:37 -0700 Subject: [PATCH 082/120] Update BUILDDIR envvar in docs/Makefile Signed-off-by: Jono Yang --- docs/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Makefile b/docs/Makefile index 788b0396..94f686b2 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -7,7 +7,7 @@ SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SPHINXAUTOBUILD = sphinx-autobuild SOURCEDIR = source -BUILDDIR = build +BUILDDIR = _build # Put it first so that "make" without argument is like "make help". help: From 5b0f4d6b4079719caa9ed97efb2ba776bc2bbac1 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 1 Apr 2025 14:42:52 +0200 Subject: [PATCH 083/120] Fix doc line length Signed-off-by: Philippe Ombredanne --- docs/source/contribute/contrib_doc.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/contribute/contrib_doc.rst b/docs/source/contribute/contrib_doc.rst index dee9296d..2a719a52 100644 --- a/docs/source/contribute/contrib_doc.rst +++ b/docs/source/contribute/contrib_doc.rst @@ -97,7 +97,7 @@ What is Checked? ^^^^^^^^^^^^^^^^ PyCQA is an Organization for code quality tools (and plugins) for the Python programming language. -Doc8 is a sub-project of the same Organization. Refer this +Doc8 is a sub-project of the same Organization. Refer this `README `_ for more details. What is checked: @@ -263,7 +263,7 @@ Converting from Markdown ------------------------ If you want to convert a ``.md`` file to a ``.rst`` file, this -`tool `_ does it pretty well. +`tool `_ does it pretty well. You will still have to clean up and check for errors as this contains a lot of bugs. But this is definitely better than converting everything by yourself. From e776fef5ad595378752d39425109a4cbd2cb5175 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 1 Apr 2025 14:49:40 +0200 Subject: [PATCH 084/120] Format code Signed-off-by: Philippe Ombredanne --- etc/scripts/update_skeleton.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/etc/scripts/update_skeleton.py b/etc/scripts/update_skeleton.py index 5705fc43..374c06f2 100644 --- a/etc/scripts/update_skeleton.py +++ b/etc/scripts/update_skeleton.py @@ -15,7 +15,7 @@ import click -ABOUTCODE_PUBLIC_REPO_NAMES=[ +ABOUTCODE_PUBLIC_REPO_NAMES = [ "aboutcode-toolkit", "ahocode", "bitcode", @@ -87,7 +87,9 @@ def update_skeleton_files(repo_names=ABOUTCODE_PUBLIC_REPO_NAMES): os.chdir(work_dir_path / repo_name) # Add skeleton as an origin - subprocess.run(["git", "remote", "add", "skeleton", "git@github.com:aboutcode-org/skeleton.git"]) + subprocess.run( + ["git", "remote", "add", "skeleton", "git@github.com:aboutcode-org/skeleton.git"] + ) # Fetch skeleton files subprocess.run(["git", "fetch", "skeleton"]) From 2a43f4cdc8105473b279eea873db00addaa14551 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 1 Apr 2025 14:59:05 +0200 Subject: [PATCH 085/120] Correct supported runner on Azure See for details: https://learn.microsoft.com/en-us/azure/devops/pipelines/agents/hosted?view=azure-devops&tabs=yaml macOS ARM images do not seem to be supported there Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 80ae45b1..fb03c090 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -26,14 +26,6 @@ jobs: - template: etc/ci/azure-posix.yml parameters: job_name: macos13_cpython - image_name: macOS-13-xlarge - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos13_cpython_arm64 image_name: macOS-13 python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: @@ -42,14 +34,6 @@ jobs: - template: etc/ci/azure-posix.yml parameters: job_name: macos14_cpython - image_name: macOS-14-large - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos14_cpython_arm64 image_name: macOS-14 python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: @@ -63,14 +47,6 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos15_cpython_arm64 - image_name: macOS-15-large - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-win.yml parameters: job_name: win2019_cpython From 4a15550b7bcea5ec949a5049fe1a501d3bb888ff Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 1 Apr 2025 19:34:14 +0200 Subject: [PATCH 086/120] Add code checks to CI Remove running "make check" as a test Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 8 ++++++ tests/test_skeleton_codestyle.py | 44 -------------------------------- 2 files changed, 8 insertions(+), 44 deletions(-) delete mode 100644 tests/test_skeleton_codestyle.py diff --git a/azure-pipelines.yml b/azure-pipelines.yml index fb03c090..ad18b28a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,6 +7,14 @@ jobs: + - template: etc/ci/azure-posix.yml + parameters: + job_name: run_code_checks + image_name: ubuntu-24.04 + python_versions: ['3.12'] + test_suites: + all: make check + - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu22_cpython diff --git a/tests/test_skeleton_codestyle.py b/tests/test_skeleton_codestyle.py deleted file mode 100644 index 6060c085..00000000 --- a/tests/test_skeleton_codestyle.py +++ /dev/null @@ -1,44 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import configparser -import subprocess -import unittest - - -class BaseTests(unittest.TestCase): - def test_skeleton_codestyle(self): - # This test shouldn't run in proliferated repositories. - - # TODO: update with switch to pyproject.toml - setup_cfg = configparser.ConfigParser() - setup_cfg.read("setup.cfg") - if setup_cfg["metadata"]["name"] != "skeleton": - return - - commands = [ - ["venv/bin/ruff", "check"], - ["venv/bin/ruff", "format", "--check"], - ] - command = None - try: - for command in commands: - subprocess.check_output(command) # noqa: S603 - except subprocess.CalledProcessError as e: - print("===========================================================") - print(e.output) - print("===========================================================") - raise Exception( - f"Code style and linting command check failed: {' '.join(command)!r}.\n" - "You can check and format the code using:\n" - " make valid\n", - "OR:\n ruff format\n", - " ruff check --fix\n", - e.output, - ) from e From b2d7512735ca257088d2cac4b55590fc5d7b20b4 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Tue, 1 Apr 2025 20:15:18 +0200 Subject: [PATCH 087/120] Revert support for Python 3.13 This is not yet supported everywhere Signed-off-by: Philippe Ombredanne --- azure-pipelines.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ad18b28a..7a2d4d9b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,7 +19,7 @@ jobs: parameters: job_name: ubuntu22_cpython image_name: ubuntu-22.04 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -27,7 +27,7 @@ jobs: parameters: job_name: ubuntu24_cpython image_name: ubuntu-24.04 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -35,7 +35,7 @@ jobs: parameters: job_name: macos13_cpython image_name: macOS-13 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -43,7 +43,7 @@ jobs: parameters: job_name: macos14_cpython image_name: macOS-14 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -51,7 +51,7 @@ jobs: parameters: job_name: macos15_cpython image_name: macOS-15 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -59,7 +59,7 @@ jobs: parameters: job_name: win2019_cpython image_name: windows-2019 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -67,7 +67,7 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -75,6 +75,6 @@ jobs: parameters: job_name: win2025_cpython image_name: windows-2025 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ['3.9', '3.10', '3.11', '3.12'] test_suites: all: venv\Scripts\pytest -n 2 -vvs From 2e3464b79811bcf505d93692da2418e2444150ed Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 16:52:32 +0200 Subject: [PATCH 088/120] Ignore local .env file Signed-off-by: Philippe Ombredanne --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8a93c94d..4818bb3a 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,4 @@ tcl # Ignore Jupyter Notebook related temp files .ipynb_checkpoints/ /.ruff_cache/ +.env \ No newline at end of file From d4af79f0da82ab0d16dcf60b363a6a5290cd9403 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 16:54:29 +0200 Subject: [PATCH 089/120] Add correct extras for documentation Signed-off-by: Philippe Ombredanne --- .readthedocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 8ab23688..7e399c8a 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -26,4 +26,4 @@ python: - method: pip path: . extra_requirements: - - docs + - testing From 49bfd37c7273f2118d35585c67e53f0cf7642f43 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 16:58:37 +0200 Subject: [PATCH 090/120] Improve MANIFEST Signed-off-by: Philippe Ombredanne --- MANIFEST.in | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index ef3721e8..0f197075 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,6 @@ graft src +graft docs +graft etc include *.LICENSE include NOTICE @@ -6,10 +8,18 @@ include *.ABOUT include *.toml include *.yml include *.rst +include *.png include setup.* include configure* include requirements* -include .git* +include .dockerignore +include .gitignore +include .readthedocs.yml +include manage.py +include Dockerfile* +include Makefile +include MANIFEST.in -global-exclude *.py[co] __pycache__ *.*~ +include .VERSION +global-exclude *.py[co] __pycache__ *.*~ From 5bc987a16cb3ae0f6de101a9c9e277df431f4317 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 17:12:58 +0200 Subject: [PATCH 091/120] Improve cleaning on POSIX Signed-off-by: Philippe Ombredanne --- configure | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure b/configure index 83fd2035..3dd9a0ab 100755 --- a/configure +++ b/configure @@ -35,7 +35,7 @@ DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constrai VIRTUALENV_DIR=venv # Cleanable files and directories to delete with the --clean option -CLEANABLE="build dist venv .cache .eggs" +CLEANABLE="build dist venv .cache .eggs *.egg-info docs/_build/ pip-selfcheck.json" # extra arguments passed to pip PIP_EXTRA_ARGS=" " @@ -167,6 +167,7 @@ clean() { for cln in $CLEANABLE; do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; done + find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete set +e exit } From 887779a9bd36650ffc6751f0069b492e80dd2f08 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 17:19:44 +0200 Subject: [PATCH 092/120] Rename dev extra to "dev" Instead of testing ... and update references accordingly Signed-off-by: Philippe Ombredanne --- .readthedocs.yml | 2 +- Makefile | 7 ++++++- configure | 2 +- configure.bat | 2 +- setup.cfg | 2 +- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index 7e399c8a..683f3a82 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -26,4 +26,4 @@ python: - method: pip path: . extra_requirements: - - testing + - dev diff --git a/Makefile b/Makefile index 413399e5..3041547b 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,13 @@ PYTHON_EXE?=python3 VENV=venv ACTIVATE?=. ${VENV}/bin/activate; + +conf: + @echo "-> Install dependencies" + ./configure + dev: - @echo "-> Configure the development envt." + @echo "-> Configure and install development dependencies" ./configure --dev doc8: diff --git a/configure b/configure index 3dd9a0ab..5ef0e063 100755 --- a/configure +++ b/configure @@ -29,7 +29,7 @@ CLI_ARGS=$1 # Requirement arguments passed to pip and used by default or with --dev. REQUIREMENTS="--editable . --constraint requirements.txt" -DEV_REQUIREMENTS="--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" +DEV_REQUIREMENTS="--editable .[dev] --constraint requirements.txt --constraint requirements-dev.txt" # where we create a virtualenv VIRTUALENV_DIR=venv diff --git a/configure.bat b/configure.bat index 18b37038..3e9881fb 100644 --- a/configure.bat +++ b/configure.bat @@ -27,7 +27,7 @@ @rem # Requirement arguments passed to pip and used by default or with --dev. set "REQUIREMENTS=--editable . --constraint requirements.txt" -set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" +set "DEV_REQUIREMENTS=--editable .[dev] --constraint requirements.txt --constraint requirements-dev.txt" @rem # where we create a virtualenv set "VIRTUALENV_DIR=venv" diff --git a/setup.cfg b/setup.cfg index ad8e0d8f..99ba2607 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,7 +48,7 @@ where = src [options.extras_require] -testing = +dev = pytest >= 6, != 7.0.0 pytest-xdist >= 2 aboutcode-toolkit >= 7.0.2 From 209231f0d27de0b0cfcc51eeb0fbaf9393d3df1c Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 17:50:44 +0200 Subject: [PATCH 093/120] Add more excludes from tests Signed-off-by: Philippe Ombredanne --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c9e67720..bcca1a8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,6 @@ norecursedirs = [ "dist", "build", "_build", - "dist", "etc", "local", "ci", @@ -34,7 +33,9 @@ norecursedirs = [ "thirdparty", "tmp", "venv", + ".venv", "tests/data", + "*/tests/test_data", ".eggs", "src/*/data", "tests/*/data" From 47bce2da33db6b3ce3bb16831ddca89a65494e23 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 17:54:23 +0200 Subject: [PATCH 094/120] Do not lint django migrations Signed-off-by: Philippe Ombredanne --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bcca1a8a..d79574ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ include = [ "docs/**/*.py", "*.py", "." + ] # ignore test data and testfiles: they should never be linted nor formatted exclude = [ @@ -78,9 +79,10 @@ exclude = [ # vulnerablecode, fetchcode "**/tests/*/test_data/**/*", "**/tests/test_data/**/*", +# django migrations + "**/migrations/**/*" ] - [tool.ruff.lint] # Rules: https://docs.astral.sh/ruff/rules/ select = [ From 5025cfb59f0555bf4b40cd75e75ce41188e19e11 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 17:58:07 +0200 Subject: [PATCH 095/120] Add README.rst to list of "license files" Signed-off-by: Philippe Ombredanne --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 99ba2607..e5b56dad 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,6 +28,7 @@ license_files = AUTHORS.rst CHANGELOG.rst CODE_OF_CONDUCT.rst + README.rst [options] package_dir = From 548a72eac69e4400e4b01f22941d38fe1cb4648d Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 18:59:08 +0200 Subject: [PATCH 096/120] Use Python 3.9 as lowest suupported version Signed-off-by: Philippe Ombredanne --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index e5b56dad..a9c5dbc6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,8 @@ license_files = README.rst [options] +python_requires = >=3.9 + package_dir = =src packages = find: @@ -39,7 +41,6 @@ zip_safe = false setup_requires = setuptools_scm[toml] >= 4 -python_requires = >=3.8 install_requires = From 3d256b4ac7976b46c23424e86bb62a38f0e4a095 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 19:01:22 +0200 Subject: [PATCH 097/120] Drop pycodestyle Not used anymore Signed-off-by: Philippe Ombredanne --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index a9c5dbc6..6d0b6488 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,7 +54,6 @@ dev = pytest >= 6, != 7.0.0 pytest-xdist >= 2 aboutcode-toolkit >= 7.0.2 - pycodestyle >= 2.8.0 twine ruff Sphinx>=5.0.2 From 645052974bf7e6f45c1e55a24a2acaa0cee24523 Mon Sep 17 00:00:00 2001 From: Philippe Ombredanne Date: Wed, 2 Apr 2025 19:26:17 +0200 Subject: [PATCH 098/120] Bump pytest minimal version Signed-off-by: Philippe Ombredanne --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 6d0b6488..69f850ca 100644 --- a/setup.cfg +++ b/setup.cfg @@ -51,7 +51,7 @@ where = src [options.extras_require] dev = - pytest >= 6, != 7.0.0 + pytest >= 7.0.1 pytest-xdist >= 2 aboutcode-toolkit >= 7.0.2 twine From 49408c25affc81c8bbcd1567be1c52f254a6f208 Mon Sep 17 00:00:00 2001 From: Theodore Aptekarev Date: Mon, 19 May 2025 14:42:46 +0300 Subject: [PATCH 099/120] Fix using incorrect hidden property on ProgressBar instance Signed-off-by: Theodore Aptekarev Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 4 ++-- requirements-dev.txt | 6 +++--- requirements.txt | 27 ++++++++++++++------------- setup.cfg | 5 ++--- src/commoncode/cliutils.py | 6 +++--- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a3fc9e8e..5d09bda0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -68,9 +68,9 @@ jobs: python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] test_suites: click_versions: | - for clk_ver in 8.1.7 8.1.6 8.1.5 8.1.4 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7; + for clk_ver in 8.2.0 8.2.1; do - pip install click==$clk_ver; + venv/bin/pip install --force-reinstall click==$clk_ver; venv/bin/pytest -vvs tests/test_cliutils_progressbar.py; done diff --git a/requirements-dev.txt b/requirements-dev.txt index 46455179..c2d9aa0f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -aboutcode-toolkit==7.2.0 +aboutcode-toolkit==11.1.1 black==23.1.0 bleach==6.0.0 boolean.py==4.0 @@ -6,10 +6,10 @@ cffi==1.15.1 cryptography==39.0.1 docutils==0.19 et-xmlfile==1.1.0 -exceptiongroup==1.1.0 execnet==1.9.0 importlib-metadata==6.0.0 iniconfig==2.0.0 +isort==6.0.1 jaraco.classes==3.2.3 jeepney==0.8.0 jinja2==3.1.2 @@ -27,6 +27,7 @@ pathspec==0.11.0 pkginfo==1.9.6 platformdirs==3.0.0 pluggy==1.0.0 +pycodestyle==2.13.0 pycparser==2.21 pygments==2.14.0 pytest==7.2.1 @@ -37,7 +38,6 @@ rfc3986==2.0.0 rich==13.3.1 secretstorage==3.3.3 six==1.16.0 -tomli==2.0.1 twine==4.0.2 webencodings==0.5.1 zipp==3.14.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 562ae0ba..c48b99e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,16 @@ -attrs==22.2.0 -beautifulsoup4==4.13.3 -certifi==2022.12.7 +attrs==25.3.0 +beautifulsoup4==4.13.4 +certifi==2025.4.26 chardet==5.2.0 -click==8.1.3 -idna==3.4 -pip==23.0 -PyYAML==6.0 -requests==2.28.2 -saneyaml==0.6.0 -setuptools==67.1.0 -soupsieve==2.4 +charset-normalizer==3.4.2 +click==8.2.1 +idna==3.10 +pip==25.1.1 +PyYAML==6.0.2 +requests==2.32.2 +saneyaml==0.6.1 +setuptools==80.3.1 +soupsieve==2.7 text-unidecode==1.3 -urllib3==1.26.14 -wheel==0.38.4 +typing_extensions==4.13.2 +urllib3==1.26.20 diff --git a/setup.cfg b/setup.cfg index 063b149e..c172c365 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,7 @@ python_requires = >=3.9 install_requires = attrs >= 18.1, !=20.1.0 Beautifulsoup4[chardet] >= 4.13.0 - click >= 6.7, !=7.0 + click >= 8.2.0 requests[use_chardet_on_py3] >= 2.7.0 saneyaml >= 0.5.2 text_unidecode >= 1.0 @@ -56,7 +56,7 @@ where = src testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 - aboutcode-toolkit >= 7.0.2 + aboutcode-toolkit >=11.1.1 pycodestyle >= 2.8.0 twine black @@ -70,4 +70,3 @@ docs = sphinx-autobuild sphinx-rtd-dark-mode>=1.3.0 sphinx-copybutton - diff --git a/src/commoncode/cliutils.py b/src/commoncode/cliutils.py index 8ec419c4..a1d2723e 100644 --- a/src/commoncode/cliutils.py +++ b/src/commoncode/cliutils.py @@ -177,7 +177,7 @@ def make_step(self, n_steps): # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 def generator(self): - if self.is_hidden: + if self.hidden: yield from self.iter else: for rv in self.iter: @@ -196,7 +196,7 @@ class EnhancedProgressBar(DebuggedProgressBar): """ def render_progress(self): - if not self.is_hidden: + if not self.hidden: return super(EnhancedProgressBar, self).render_progress() @@ -217,7 +217,7 @@ class ProgressLogger(ProgressBar): def __init__(self, *args, **kwargs): super(ProgressLogger, self).__init__(*args, **kwargs) - self.is_hidden = False + self.hidden = False def render_progress(self): line = self.format_progress_line() From 48001c36ba0161a70efe053bece239e9572c02e5 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 11 Jun 2025 17:25:40 +0530 Subject: [PATCH 100/120] Fix compatibility issues from click >= 8.2.0 Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 6 +++--- requirements-dev.txt | 2 ++ requirements.txt | 4 +++- setup.cfg | 8 +++++--- src/commoncode/cliutils.py | 21 ++++++++++++++++----- 5 files changed, 29 insertions(+), 12 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5d09bda0..c4743d99 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,7 +37,7 @@ jobs: parameters: job_name: macos14_cpython image_name: macOS-14 - python_versions: ['3.9', '3.10', '3.11', '3.12'] + python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -68,9 +68,9 @@ jobs: python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] test_suites: click_versions: | - for clk_ver in 8.2.0 8.2.1; + for clk_ver in 8.2.0 8.2.1 8.1.8 8.1.7 8.1.6 8.1.5 8.1.4 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7; do - venv/bin/pip install --force-reinstall click==$clk_ver; + venv/bin/pip install click==$clk_ver; venv/bin/pytest -vvs tests/test_cliutils_progressbar.py; done diff --git a/requirements-dev.txt b/requirements-dev.txt index c2d9aa0f..0dc33788 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,6 +6,7 @@ cffi==1.15.1 cryptography==39.0.1 docutils==0.19 et-xmlfile==1.1.0 +exceptiongroup==1.1.0 execnet==1.9.0 importlib-metadata==6.0.0 iniconfig==2.0.0 @@ -38,6 +39,7 @@ rfc3986==2.0.0 rich==13.3.1 secretstorage==3.3.3 six==1.16.0 +tomli==2.0.1 twine==4.0.2 webencodings==0.5.1 zipp==3.14.0 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c48b99e4..ec240590 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,8 @@ beautifulsoup4==4.13.4 certifi==2025.4.26 chardet==5.2.0 charset-normalizer==3.4.2 -click==8.2.1 +click==8.2.1;python_version>='3.10' +click==8.1.8;python_version<'3.10' idna==3.10 pip==25.1.1 PyYAML==6.0.2 @@ -14,3 +15,4 @@ soupsieve==2.7 text-unidecode==1.3 typing_extensions==4.13.2 urllib3==1.26.20 +wheel==0.38.4 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index c172c365..852bf808 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,9 +40,11 @@ setup_requires = setuptools_scm[toml] >= 4 python_requires = >=3.9 install_requires = - attrs >= 18.1, !=20.1.0 + attrs >= 18.1,!=20.1.0;python_version<'3.11' + attrs >= 22.1.0;python_version>='3.11' Beautifulsoup4[chardet] >= 4.13.0 - click >= 8.2.0 + click >= 6.7, !=7.0;python_version<'3.10' + click >= 8.2.0;python_version>='3.10' requests[use_chardet_on_py3] >= 2.7.0 saneyaml >= 0.5.2 text_unidecode >= 1.0 @@ -56,7 +58,7 @@ where = src testing = pytest >= 6, != 7.0.0 pytest-xdist >= 2 - aboutcode-toolkit >=11.1.1 + aboutcode-toolkit >= 11.1.1 pycodestyle >= 2.8.0 twine black diff --git a/src/commoncode/cliutils.py b/src/commoncode/cliutils.py index a1d2723e..026e26bd 100644 --- a/src/commoncode/cliutils.py +++ b/src/commoncode/cliutils.py @@ -164,9 +164,20 @@ def format_options(self, ctx, formatter): formatter.write_dl(sorted_records) +class CompatProgressBar(ProgressBar): + # TODO Remove when dropping support for Python 3.9 or Click 8.1. + @property + def is_hidden(self) -> bool: + return self.hidden + + @is_hidden.setter + def is_hidden(self, value: bool) -> None: + self.hidden = value + + # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 -class DebuggedProgressBar(ProgressBar): +class DebuggedProgressBar(CompatProgressBar): # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 def make_step(self, n_steps): @@ -177,7 +188,7 @@ def make_step(self, n_steps): # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 def generator(self): - if self.hidden: + if self.is_hidden: yield from self.iter else: for rv in self.iter: @@ -196,11 +207,11 @@ class EnhancedProgressBar(DebuggedProgressBar): """ def render_progress(self): - if not self.hidden: + if not self.is_hidden: return super(EnhancedProgressBar, self).render_progress() -class ProgressLogger(ProgressBar): +class ProgressLogger(CompatProgressBar): """ A subclass of Click ProgressBar providing a verbose line-by-line progress reporting. @@ -217,7 +228,7 @@ class ProgressLogger(ProgressBar): def __init__(self, *args, **kwargs): super(ProgressLogger, self).__init__(*args, **kwargs) - self.hidden = False + self.is_hidden = False def render_progress(self): line = self.format_progress_line() From 5ba6bc5ed734921d0a088922fc651696813a9278 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 11 Jun 2025 18:52:49 +0530 Subject: [PATCH 101/120] Add @piiq in AUTHORS.rst Signed-off-by: Ayan Sinha Mahapatra --- AUTHORS.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.rst b/AUTHORS.rst index d248703c..98d13aa7 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -11,4 +11,5 @@ The following organizations or individuals have contributed to this repo: - Saravanan G @SaravananOffl - Sebastian Schuberth @sschuberth - Steven Esser @majurg +- Theodore Aptekarev @piiq - Thomas Druez @tdruez From e2db62f776857cae39fe4eb313aec67aa7152ef6 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 11 Jun 2025 20:08:00 +0530 Subject: [PATCH 102/120] Update CHANGELOG for release v32.3.0 Signed-off-by: Ayan Sinha Mahapatra --- CHANGELOG.rst | 6 ++++++ setup.cfg | 11 ++--------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 64d94ee1..8565d39e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Release notes ============= +Version 32.3.0 - (2025-03-06) +----------------------------- + +- migrate ``os.listdir()`` to ``os.scandir()`` to increase performance +- Fix checksums for empty files + Version 32.2.1 - (2025-03-06) ----------------------------- diff --git a/setup.cfg b/setup.cfg index d1967363..667e65e8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,11 +1,4 @@ [metadata] -license_files = - apache-2.0.LICENSE - NOTICE - AUTHORS.rst - CHANGELOG.rst - commoncode.ABOUT - README.rst name = commoncode author = nexB. Inc. and others author_email = info@aboutcode.org @@ -28,8 +21,6 @@ keywords = utilities scancode-toolkit - -[options] license_files = apache-2.0.LICENSE NOTICE @@ -37,7 +28,9 @@ license_files = CHANGELOG.rst CODE_OF_CONDUCT.rst README.rst + commoncode.ABOUT +[options] python_requires = >=3.9 package_dir = =src From 5d8bd13ce5176d511fa854cf9e0a79bcc5fe30e9 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 11 Jun 2025 20:30:41 +0530 Subject: [PATCH 103/120] Fix style check failures Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/docs-ci.yml | 2 +- CHANGELOG.rst | 8 +++++--- src/commoncode/command.py | 2 +- src/commoncode/datautils.py | 18 +++++++++--------- src/commoncode/dict_utils.py | 1 + src/commoncode/fetch.py | 2 +- src/commoncode/fileutils.py | 8 ++++---- src/commoncode/resource.py | 6 ++---- src/commoncode/testcase.py | 5 ++--- src/commoncode/text.py | 6 +++--- src/commoncode/urn.py | 1 + tests/test_fileutils.py | 4 ---- 12 files changed, 30 insertions(+), 33 deletions(-) diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml index 0f411df5..186c3cd8 100644 --- a/.github/workflows/docs-ci.yml +++ b/.github/workflows/docs-ci.yml @@ -30,4 +30,4 @@ jobs: run: make doc8 - name: Check for Code style errors - run: make check-ci + run: make check diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8565d39e..54a49738 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,8 +4,9 @@ Release notes Version 32.3.0 - (2025-03-06) ----------------------------- -- migrate ``os.listdir()`` to ``os.scandir()`` to increase performance -- Fix checksums for empty files +- Fix ``click`` compatibililty issues. + https://github.com/aboutcode-org/commoncode/pull/89 + https://github.com/aboutcode-org/commoncode/pull/86 Version 32.2.1 - (2025-03-06) ----------------------------- @@ -41,7 +42,8 @@ Version 32.0.0 - (2024-09-05) Version 31.2.1 - (2024-05-16) ----------------------------- -- Remove ``commoncode.system.get_etc_os_release_info`` and replace it with ``commoncode.distro_os_release_parser``. +- Remove ``commoncode.system.get_etc_os_release_info`` and replace it with + ``commoncode.distro_os_release_parser``. Version 31.2.0 - (2024-05-16) diff --git a/src/commoncode/command.py b/src/commoncode/command.py index 419da26d..59da39c3 100644 --- a/src/commoncode/command.py +++ b/src/commoncode/command.py @@ -129,9 +129,9 @@ def execute2( log=TRACE, ): """ - DEPRECATED: DO NOT USE. Use execute() instead Run a `cmd_loc` command with the `args` arguments list and return the return code, the stdout and stderr. + DEPRECATED: DO NOT USE. Use execute() instead To avoid RAM exhaustion, always write stdout and stderr streams to files. diff --git a/src/commoncode/datautils.py b/src/commoncode/datautils.py index db67e219..7570fb57 100644 --- a/src/commoncode/datautils.py +++ b/src/commoncode/datautils.py @@ -33,7 +33,7 @@ def attribute( metadata=None, ): # NOQA """ - A generic attribute with help metadata and that is not included in the + Return a generic attribute with help metadata and that is not included in the representation by default. """ metadata = metadata or dict() @@ -67,7 +67,7 @@ def Boolean( help=None, ): # NOQA """ - A boolean attribute. + Return a boolean attribute. """ return attribute( default=default, @@ -94,7 +94,7 @@ def TriBoolean( help=None, ): # NOQA """ - A tri-boolean attribute with possible values of None, True and False. + Return a tri-boolean attribute with possible values of None, True and False. """ return attribute( default=default, @@ -121,7 +121,7 @@ def String( help=None, ): # NOQA """ - A string attribute. + Return a string attribute. """ return attribute( default=default, @@ -148,7 +148,7 @@ def Integer( help=None, ): # NOQA """ - An integer attribute. + Return an integer attribute. """ converter = converter or attr.converters.optional(int) return attribute( @@ -176,7 +176,7 @@ def Float( help=None, ): # NOQA """ - A float attribute. + Return a float attribute. """ return attribute( default=default, @@ -204,7 +204,7 @@ def List( help=None, ): # NOQA """ - A list attribute: the optional item_type defines the type of items it stores. + Return a list attribute: the optional item_type defines the type of items it stores. """ if default is attr.NOTHING: default = attr.Factory(list) @@ -235,7 +235,7 @@ def Mapping( label=None, ): # NOQA """ - A mapping attribute: the optional value_type defines the type of values it + Return a mapping attribute: the optional value_type defines the type of values it stores. The key is always a string. Notes: in Python 2 the type is Dict as there is no typing available for @@ -274,7 +274,7 @@ def Date( help=None, ): # NOQA """ - A date attribute. It always serializes to an ISO date string. + Return a date attribute. It always serializes to an ISO date string. Behavior is TBD and for now this is exactly a string. """ return String( diff --git a/src/commoncode/dict_utils.py b/src/commoncode/dict_utils.py index 1473f6da..c35c3d8b 100644 --- a/src/commoncode/dict_utils.py +++ b/src/commoncode/dict_utils.py @@ -18,6 +18,7 @@ def sparsify(d): Example: >>> sparsify({1: 3, 4: 5}) {1: 3, 4: 5} + """ e = d.copy() d.update(e) diff --git a/src/commoncode/fetch.py b/src/commoncode/fetch.py index 3db98c27..cf3d6dad 100644 --- a/src/commoncode/fetch.py +++ b/src/commoncode/fetch.py @@ -54,7 +54,7 @@ def download_url(url, file_name=None, verify=True, timeout=10): def ping_url(url): """ - Returns True is `url` is reachable. + Return True is `url` is reachable. """ try: from urlib.request import urlopen diff --git a/src/commoncode/fileutils.py b/src/commoncode/fileutils.py index 8d2a1e52..b8d9001b 100644 --- a/src/commoncode/fileutils.py +++ b/src/commoncode/fileutils.py @@ -70,7 +70,7 @@ def create_dir(location): if os.path.exists(location): if not os.path.isdir(location): - err = "Cannot create directory: existing file " "in the way " "%(location)s." + err = "Cannot create directory: existing file in the way %(location)s." raise OSError(err % locals()) else: # may fail on win if the path is too long @@ -140,7 +140,7 @@ def prepare_path(pth): def is_posixpath(location): - """ + r""" Return True if the `location` path is likely a POSIX-like path using POSIX path separators (slash or "/")or has no path separator. @@ -164,7 +164,7 @@ def is_posixpath(location): def as_posixpath(location): - """ + r""" Return a POSIX-like path using POSIX path separators (slash or "/") for a `location` path. This converts Windows paths to look like POSIX paths: Python accepts gracefully POSIX paths on Windows. @@ -174,7 +174,7 @@ def as_posixpath(location): def as_winpath(location): - """ + r""" Return a Windows-like path using Windows path separators (backslash or "\") for a `location` path. """ diff --git a/src/commoncode/resource.py b/src/commoncode/resource.py index d66476fb..64dbe7cc 100644 --- a/src/commoncode/resource.py +++ b/src/commoncode/resource.py @@ -120,8 +120,6 @@ def depth_walk( directory recursively up to ``max_depth`` path segments extending from the ``root_location``. The behaviour is similar of ``os.walk``. - Arguments: - - root_location: Absolute, normalized path for the directory to be walked - max_depth: positive integer for fixed depth limit. 0 for no limit. - skip_ignored: Callback function that takes a location as argument and @@ -829,7 +827,7 @@ def _dump_resource(self, resource): if not cache_location: raise TypeError( - "Resource cannot be dumped to disk and is used only" f"in memory: {resource}" + f"Resource cannot be dumped to disk and is used onlyin memory: {resource}" ) # TODO: consider messagepack or protobuf for compact/faster processing? @@ -1173,7 +1171,7 @@ def type(self, value): @classmethod def build_path(cls, root_location, location): - """ + r""" Return a POSIX path string (using "/" separators) of ``location`` relative to ``root_location`. Both locations are absolute native locations. The returned path has no leading and trailing slashes. The first segment diff --git a/src/commoncode/testcase.py b/src/commoncode/testcase.py index b24d4d1c..525bf666 100644 --- a/src/commoncode/testcase.py +++ b/src/commoncode/testcase.py @@ -69,8 +69,7 @@ def get_test_loc( if not path.exists(test_data_dir): raise IOError( - "[Errno 2] No such directory: test_data_dir not found:" - " '%(test_data_dir)s'" % locals() + "[Errno 2] No such directory: test_data_dir not found: '%(test_data_dir)s'" % locals() ) tpath = to_os_native_path(test_path) @@ -78,7 +77,7 @@ def get_test_loc( if must_exist and not path.exists(test_loc): raise IOError( - "[Errno 2] No such file or directory: " "test_path not found: '%(test_loc)s'" % locals() + "[Errno 2] No such file or directory: test_path not found: '%(test_loc)s'" % locals() ) return test_loc diff --git a/src/commoncode/text.py b/src/commoncode/text.py index 20c11793..2c39c6b7 100644 --- a/src/commoncode/text.py +++ b/src/commoncode/text.py @@ -27,7 +27,7 @@ def lines(s): - """ + r""" Split a string in lines using the following conventions: - a line ending \r\n or \n is a separator and yields a new list element - empty lines or lines with only white spaces are not returned. @@ -54,7 +54,7 @@ def nopunc(): def nopunctuation(text): """ - Replaces any non alphanum symbol (i.e. punctuation) in text with space. + Replace any non alphanum symbol (i.e. punctuation) in text with space. Preserve the characters offsets by replacing punctuation with spaces. Warning: this also drops line endings. """ @@ -81,7 +81,7 @@ def unixlinesep(text, preserve=False): def nolinesep(text): """ - Removes line separators, replacing them with spaces. + Remove line separators, replacing them with spaces. """ if not isinstance(text, str): text = as_unicode(text) diff --git a/src/commoncode/urn.py b/src/commoncode/urn.py index 32dce797..04922024 100644 --- a/src/commoncode/urn.py +++ b/src/commoncode/urn.py @@ -63,6 +63,7 @@ urn:dje:component:dropbear: The product object type syntax is the same as the component syntax. + """ from urllib.parse import quote_plus diff --git a/tests/test_fileutils.py b/tests/test_fileutils.py index 6627c216..e3913b79 100644 --- a/tests/test_fileutils.py +++ b/tests/test_fileutils.py @@ -28,10 +28,6 @@ @skip("Somehow permissions tests do not work OK yet on Python 3") class TestPermissionsDeletions(FileBasedTesting): - """ - This is failing for now on Python 3 - """ - test_data_dir = os.path.join(os.path.dirname(__file__), "data") def test_delete_unwritable_directory_and_files(self): From 3d0e0615a2265009b73bb0c0e3cf13a3b9229b13 Mon Sep 17 00:00:00 2001 From: Adrian Braemer Date: Fri, 17 Oct 2025 16:36:25 +0200 Subject: [PATCH 104/120] fix: handle paths with non-utf-8 bytes Signed-off-by: Adrian Braemer --- src/commoncode/resource.py | 2 +- tests/test_resource.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/commoncode/resource.py b/src/commoncode/resource.py index 64dbe7cc..d19e2da8 100644 --- a/src/commoncode/resource.py +++ b/src/commoncode/resource.py @@ -389,7 +389,7 @@ def _get_resource_cache_location(self, path, create_dirs=False): path = clean_path(path) # for the cached file name, we use an md5 of the path to avoid things being too long - resid = str(md5(path.encode("utf-8")).hexdigest()) + resid = str(md5(path.encode("utf-8", "surrogateescape")).hexdigest()) cache_sub_dir, cache_file_name = resid[-2:], resid parent = join(self.cache_dir, cache_sub_dir) diff --git a/tests/test_resource.py b/tests/test_resource.py index 9711b660..70bc18b0 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -694,6 +694,11 @@ def test_codebase_cache_default(self): codebase.save_resource(child) child_2 = codebase.get_resource(path=child.path) assert child_2 == child + + def test_codebase_cache_handles_non_utf8_path(self): + test_codebase = self.get_test_loc("resource/cache2") + codebase = Codebase(test_codebase) + codebase._get_resource_cache_location('resource/cache2/\udce9', create_dirs=True) def test_codebase_cache_all_in_memory(self): test_codebase = self.get_test_loc("resource/cache2") From d4b658f5bda06663d700a4a10e1e4c37b278f62e Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 21 Oct 2025 19:29:50 +0530 Subject: [PATCH 105/120] Do not set flag_value as None in click Option From click 8.3.0 flag_value is set as a sentinel value UNSET instead of None, and since we maintain a PluggableCommanLineOption subclass, we need to maintain the same default values for an Option to avoid issues. Reference: https://github.com/pallets/click/releases/tag/8.3.0 Reference: https://github.com/aboutcode-org/scancode-toolkit/issues/4572 Signed-off-by: Ayan Sinha Mahapatra --- src/commoncode/cliutils.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/commoncode/cliutils.py b/src/commoncode/cliutils.py index 026e26bd..a9273a59 100644 --- a/src/commoncode/cliutils.py +++ b/src/commoncode/cliutils.py @@ -22,6 +22,19 @@ # Tracing flags TRACE = False +try: + # Introduced in click 8.3.0 to have a sentinel value + # (https://peps.python.org/pep-0661/) for flag values + # and default values instead of None to differentiate + # between explicitly setting a `None` value and + # not setting and value. + # See https://github.com/pallets/click/pull/3030 and + # https://github.com/pallets/click/releases/tag/8.3.0 + from click.core import UNSET +except ImportError: + # to maintain compatibility with click < 8.3.0 + UNSET = None + def logger_debug(*args): pass @@ -429,7 +442,7 @@ def __init__( confirmation_prompt=False, hide_input=False, is_flag=None, - flag_value=None, + flag_value=UNSET, multiple=False, count=False, allow_from_autoenv=True, From 2cd886ba3a38897f9ef7fd4e514d87d6f1fe674e Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 21 Oct 2025 19:49:27 +0530 Subject: [PATCH 106/120] Drop python3.9 support and add python3.14 Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 4 ++-- requirements.txt | 3 +-- setup.cfg | 5 ++--- src/commoncode/cliutils.py | 15 ++------------- 4 files changed, 7 insertions(+), 20 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cd391c14..6c013b4d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -65,10 +65,10 @@ jobs: parameters: job_name: ubuntu24_test_all_supported_click_versions image_name: ubuntu-24.04 - python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: click_versions: | - for clk_ver in 8.2.0 8.2.1 8.1.8 8.1.7 8.1.6 8.1.5 8.1.4 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7; + for clk_ver in 8.3.0 8.2.0 8.2.1 8.1.8 8.1.7 8.1.6 8.1.5 8.1.4 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7; do venv/bin/pip install click==$clk_ver; venv/bin/pytest -vvs tests/test_cliutils_progressbar.py; diff --git a/requirements.txt b/requirements.txt index ec240590..0dbf9fea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,7 @@ beautifulsoup4==4.13.4 certifi==2025.4.26 chardet==5.2.0 charset-normalizer==3.4.2 -click==8.2.1;python_version>='3.10' -click==8.1.8;python_version<'3.10' +click==8.3.0 idna==3.10 pip==25.1.1 PyYAML==6.0.2 diff --git a/setup.cfg b/setup.cfg index 667e65e8..75df2cc6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,7 +31,7 @@ license_files = commoncode.ABOUT [options] -python_requires = >=3.9 +python_requires = >=3.10 package_dir = =src packages = find: @@ -44,8 +44,7 @@ install_requires = attrs >= 18.1,!=20.1.0;python_version<'3.11' attrs >= 22.1.0;python_version>='3.11' Beautifulsoup4[chardet] >= 4.13.0 - click >= 6.7, !=7.0;python_version<'3.10' - click >= 8.2.0;python_version>='3.10' + click >= 8.2.0 requests[use_chardet_on_py3] >= 2.7.0 saneyaml >= 0.5.2 text_unidecode >= 1.0 diff --git a/src/commoncode/cliutils.py b/src/commoncode/cliutils.py index a9273a59..8290db4b 100644 --- a/src/commoncode/cliutils.py +++ b/src/commoncode/cliutils.py @@ -177,20 +177,9 @@ def format_options(self, ctx, formatter): formatter.write_dl(sorted_records) -class CompatProgressBar(ProgressBar): - # TODO Remove when dropping support for Python 3.9 or Click 8.1. - @property - def is_hidden(self) -> bool: - return self.hidden - - @is_hidden.setter - def is_hidden(self, value: bool) -> None: - self.hidden = value - - # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 -class DebuggedProgressBar(CompatProgressBar): +class DebuggedProgressBar(ProgressBar): # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 def make_step(self, n_steps): @@ -224,7 +213,7 @@ def render_progress(self): return super(EnhancedProgressBar, self).render_progress() -class ProgressLogger(CompatProgressBar): +class ProgressLogger(ProgressBar): """ A subclass of Click ProgressBar providing a verbose line-by-line progress reporting. From a4f89a158e7ea8731758734919edf1d1030d2762 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 21 Oct 2025 19:52:50 +0530 Subject: [PATCH 107/120] Remove --wheel option from virtualenv configure This removes a warning seen in initial virtualenv configuration. Signed-off-by: Ayan Sinha Mahapatra --- configure | 2 +- configure.bat | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 5ef0e063..6d317d4c 100755 --- a/configure +++ b/configure @@ -110,7 +110,7 @@ create_virtualenv() { fi $PYTHON_EXECUTABLE "$VIRTUALENV_PYZ" \ - --wheel embed --pip embed --setuptools embed \ + --pip embed --setuptools embed \ --seeder pip \ --never-download \ --no-periodic-update \ diff --git a/configure.bat b/configure.bat index 3e9881fb..58ed98e0 100644 --- a/configure.bat +++ b/configure.bat @@ -110,7 +110,7 @@ if not exist "%CFG_BIN_DIR%\python.exe" ( if exist "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ( %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ^ - --wheel embed --pip embed --setuptools embed ^ + --pip embed --setuptools embed ^ --seeder pip ^ --never-download ^ --no-periodic-update ^ @@ -126,7 +126,7 @@ if not exist "%CFG_BIN_DIR%\python.exe" ( ) ) %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ^ - --wheel embed --pip embed --setuptools embed ^ + embed --pip embed --setuptools embed ^ --seeder pip ^ --never-download ^ --no-periodic-update ^ From 2e593d0fbc9d74e70bef5c159aae5948fb591ca0 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 21 Oct 2025 19:56:52 +0530 Subject: [PATCH 108/120] Replace is_hidden with hidden in ProgressBar Reference: https://github.com/aboutcode-org/scancode-toolkit/issues/4369 Signed-off-by: Ayan Sinha Mahapatra --- src/commoncode/cliutils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/commoncode/cliutils.py b/src/commoncode/cliutils.py index 8290db4b..0a5eccb1 100644 --- a/src/commoncode/cliutils.py +++ b/src/commoncode/cliutils.py @@ -190,7 +190,7 @@ def make_step(self, n_steps): # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 def generator(self): - if self.is_hidden: + if self.hidden: yield from self.iter else: for rv in self.iter: @@ -209,7 +209,7 @@ class EnhancedProgressBar(DebuggedProgressBar): """ def render_progress(self): - if not self.is_hidden: + if not self.hidden: return super(EnhancedProgressBar, self).render_progress() @@ -230,7 +230,7 @@ class ProgressLogger(ProgressBar): def __init__(self, *args, **kwargs): super(ProgressLogger, self).__init__(*args, **kwargs) - self.is_hidden = False + self.hidden = False def render_progress(self): line = self.format_progress_line() From 1af7a71958632d69fc4ffb34545c93cc81321728 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Tue, 21 Oct 2025 20:03:24 +0530 Subject: [PATCH 109/120] Update azure pipelines supported python versions Signed-off-by: Ayan Sinha Mahapatra --- azure-pipelines.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6c013b4d..1946a604 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -13,7 +13,7 @@ jobs: parameters: job_name: ubuntu22_cpython image_name: ubuntu-22.04 - python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -21,7 +21,7 @@ jobs: parameters: job_name: ubuntu24_cpython image_name: ubuntu-24.04 - python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -29,7 +29,7 @@ jobs: parameters: job_name: macos13_cpython image_name: macOS-13 - python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -37,7 +37,7 @@ jobs: parameters: job_name: macos14_cpython image_name: macOS-14 - python_versions: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: all: venv/bin/pytest -n 2 -vvs @@ -45,7 +45,7 @@ jobs: parameters: job_name: win2025_cpython image_name: windows-2025 - python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -53,7 +53,7 @@ jobs: parameters: job_name: win2022_cpython image_name: windows-2022 - python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: all: venv\Scripts\pytest -n 2 -vvs @@ -82,7 +82,7 @@ jobs: parameters: job_name: ubuntu24_cpython_latest_from_pip image_name: ubuntu-24.04 - python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: all: | venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e . @@ -93,7 +93,7 @@ jobs: parameters: job_name: win2022_cpython_latest_from_pip image_name: windows-2022 - python_versions: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: all: | venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e . From b8372311550088bcbf1d4dbcde7aec28436cd567 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 22 Oct 2025 17:10:59 +0530 Subject: [PATCH 110/120] Bump requirements to support python3.14 Signed-off-by: Ayan Sinha Mahapatra --- etc/scripts/utils_requirements.py | 2 +- requirements-dev.txt | 6 +++--- requirements.txt | 16 ++++++++-------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/etc/scripts/utils_requirements.py b/etc/scripts/utils_requirements.py index b9b2c0e7..424bed2e 100644 --- a/etc/scripts/utils_requirements.py +++ b/etc/scripts/utils_requirements.py @@ -153,7 +153,7 @@ def split_req(req): if not req: raise ValueError("req is required") # do not allow multiple constraints and tags - if not any(c in req for c in ",;"): + if any(c in req for c in ",;"): raise Exception(f"complex requirements with : or ; not supported: {req}") req = "".join(req.split()) if not any(c in req for c in comparators): diff --git a/requirements-dev.txt b/requirements-dev.txt index 0dc33788..02135b30 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ aboutcode-toolkit==11.1.1 black==23.1.0 bleach==6.0.0 boolean.py==4.0 -cffi==1.15.1 +cffi==2.0.0 cryptography==39.0.1 docutils==0.19 et-xmlfile==1.1.0 @@ -27,11 +27,11 @@ packaging==23.0 pathspec==0.11.0 pkginfo==1.9.6 platformdirs==3.0.0 -pluggy==1.0.0 +pluggy==1.6.0 pycodestyle==2.13.0 pycparser==2.21 pygments==2.14.0 -pytest==7.2.1 +pytest==8.4.2 pytest-xdist==3.2.0 readme-renderer==37.3 requests-toolbelt==0.10.1 diff --git a/requirements.txt b/requirements.txt index 0dbf9fea..665ff891 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,17 @@ -attrs==25.3.0 -beautifulsoup4==4.13.4 -certifi==2025.4.26 +attrs==25.4.0 +beautifulsoup4==4.14.2 +certifi==2025.10.5 chardet==5.2.0 -charset-normalizer==3.4.2 +charset-normalizer==3.4.4 click==8.3.0 idna==3.10 pip==25.1.1 PyYAML==6.0.2 requests==2.32.2 saneyaml==0.6.1 -setuptools==80.3.1 -soupsieve==2.7 +setuptools==80.9.0 +soupsieve==2.8 text-unidecode==1.3 -typing_extensions==4.13.2 -urllib3==1.26.20 +typing_extensions==4.15.0 +urllib3==2.5.0 wheel==0.38.4 \ No newline at end of file From 72b737bff26db96ba17b324bca31f6ad59c71839 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 22 Oct 2025 17:11:20 +0530 Subject: [PATCH 111/120] Remove special file to fix py3.14 tests Remove special file '6-FIFOTYPE' from types.tar to fix test failure as in py3.14 tarfiles extraction fails with tarfile.SpecialFileError for special files. Reference: https://github.com/aboutcode-org/commoncode/issues/88 Signed-off-by: Ayan Sinha Mahapatra --- tests/data/filetype/types.tar | Bin 109056 -> 112640 bytes tests/test_filetype.py | 1 - 2 files changed, 1 deletion(-) diff --git a/tests/data/filetype/types.tar b/tests/data/filetype/types.tar index 458dc4476efa3cb3b3507135af450adc925a1bfa..24fc04c44da43ac26ebb53a04feca6c8d1a04342 100644 GIT binary patch delta 846 zcmb7?Pfo%>6vn5dshGF`G?Hi+a{_sf&Yw#d&tOSobOQ!3LCT^r#5HYqZcJEs062jc zaHB_ny(civYeYb8QfD)1lli^xd*2tmInldQI%%Z7)k^McskLMPfLh|_ZGL5kp#-f^ zASjOkqV}4MOY=CW-t_u@mviRd`t9CT`_hlA_1*7}^ttk4#v}`xvaf{Lq|>xCT)2)z zU>pkI=Kk`r67$A%%%2Xe36Vz`S<7Gg({SWEP4tQjO)8oh8GKvB#n?QL4B5OR>Wx;S(llqQHGn_B}XdAu|n zxMiV`am>aRjJB4~s~zunSyhTEoE?!f0^=ATTpAVE|Hw=B8lUz|h2Wa`hbAM@K!$(-P{)czX9W1*7U_n#MubdU7>3)n z57O)=hGyhjx9Jm;-(;p^E#f8&h6V=a<|byQCI*HK28M=4<|YgZJ|Iu9Lp?E3&=H%9 zH#431$H>IMFq!E@3T}sWbiIafemMbq#<)tQ!Iwd zu$h<`Gbosnmxj2^blp7N{2_jb!W}0^fYz8BFesRjXAMWNZg7BOP%tn*#Ne`IdbI Date: Wed, 22 Oct 2025 17:15:33 +0530 Subject: [PATCH 112/120] Fix test failures Signed-off-by: Ayan Sinha Mahapatra --- configure.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.bat b/configure.bat index 58ed98e0..15ab7015 100644 --- a/configure.bat +++ b/configure.bat @@ -126,7 +126,7 @@ if not exist "%CFG_BIN_DIR%\python.exe" ( ) ) %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ^ - embed --pip embed --setuptools embed ^ + --pip embed --setuptools embed ^ --seeder pip ^ --never-download ^ --no-periodic-update ^ From 810f63f64c2f58a038f1e46509c48df4b131137a Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 22 Oct 2025 17:28:20 +0530 Subject: [PATCH 113/120] Revert to CompatProgressBar to support old click versions Signed-off-by: Ayan Sinha Mahapatra --- setup.cfg | 2 +- src/commoncode/cliutils.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 75df2cc6..812f1686 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,7 +44,7 @@ install_requires = attrs >= 18.1,!=20.1.0;python_version<'3.11' attrs >= 22.1.0;python_version>='3.11' Beautifulsoup4[chardet] >= 4.13.0 - click >= 8.2.0 + click >= 8.3.0 requests[use_chardet_on_py3] >= 2.7.0 saneyaml >= 0.5.2 text_unidecode >= 1.0 diff --git a/src/commoncode/cliutils.py b/src/commoncode/cliutils.py index 0a5eccb1..5b0a01e2 100644 --- a/src/commoncode/cliutils.py +++ b/src/commoncode/cliutils.py @@ -177,9 +177,20 @@ def format_options(self, ctx, formatter): formatter.write_dl(sorted_records) +class CompatProgressBar(ProgressBar): + # TODO Remove when dropping support for Click 8.1. + @property + def is_hidden(self) -> bool: + return self.hidden + + @is_hidden.setter + def is_hidden(self, value: bool) -> None: + self.hidden = value + + # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 -class DebuggedProgressBar(ProgressBar): +class DebuggedProgressBar(CompatProgressBar): # overriden and copied from Click to work around Click woes for # https://github.com/aboutcode-org/scancode-toolkit/issues/2583 def make_step(self, n_steps): @@ -213,7 +224,7 @@ def render_progress(self): return super(EnhancedProgressBar, self).render_progress() -class ProgressLogger(ProgressBar): +class ProgressLogger(CompatProgressBar): """ A subclass of Click ProgressBar providing a verbose line-by-line progress reporting. From 3b48b9bc0495fdf066f59070af1966023bb479f9 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 22 Oct 2025 17:50:29 +0530 Subject: [PATCH 114/120] Fix string quotes in test for non-UTF8 path --- tests/test_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_resource.py b/tests/test_resource.py index 70bc18b0..93e322af 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -698,7 +698,7 @@ def test_codebase_cache_default(self): def test_codebase_cache_handles_non_utf8_path(self): test_codebase = self.get_test_loc("resource/cache2") codebase = Codebase(test_codebase) - codebase._get_resource_cache_location('resource/cache2/\udce9', create_dirs=True) + codebase._get_resource_cache_location("resource/cache2/\udce9", create_dirs=True) def test_codebase_cache_all_in_memory(self): test_codebase = self.get_test_loc("resource/cache2") From 47fb4ddf2b5abedabddd22c43bc4963ffd29bd70 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 22 Oct 2025 17:53:24 +0530 Subject: [PATCH 115/120] Fix formatting in test_resource.py --- tests/test_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_resource.py b/tests/test_resource.py index 93e322af..6249ebb4 100644 --- a/tests/test_resource.py +++ b/tests/test_resource.py @@ -694,7 +694,7 @@ def test_codebase_cache_default(self): codebase.save_resource(child) child_2 = codebase.get_resource(path=child.path) assert child_2 == child - + def test_codebase_cache_handles_non_utf8_path(self): test_codebase = self.get_test_loc("resource/cache2") codebase = Codebase(test_codebase) From 495ef21837f1784f4cca9a995a085dc0dcf1ea26 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 22 Oct 2025 18:10:45 +0530 Subject: [PATCH 116/120] Bump CHANGELOG for v32.4.0 Signed-off-by: Ayan Sinha Mahapatra --- CHANGELOG.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 54a49738..4550c80e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,7 +1,19 @@ Release notes ============= -Version 32.3.0 - (2025-03-06) +Version 32.4.0 - (2025-10-22) +----------------------------- + +- Fix ``click`` 8.3.0 compatibililty issues. + https://github.com/aboutcode-org/commoncode/pull/92 + +- Drop python 3.9 support and add python 3.14 + https://github.com/aboutcode-org/commoncode/pull/92 + +- Handle paths with non-utf-8 bytes + https://github.com/aboutcode-org/commoncode/pull/91 + +Version 32.3.0 - (2025-06-11) ----------------------------- - Fix ``click`` compatibililty issues. From b404b0b8558bb388dd4c074e7c70e691cb03c814 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 22 Oct 2025 20:43:57 +0530 Subject: [PATCH 117/120] Update pypi publishing to use Trusted Publishing Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/pypi-release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index cf0579a7..294039d1 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -71,6 +71,9 @@ jobs: needs: - create-gh-release runs-on: ubuntu-24.04 + environment: pypi-publish + permissions: + id-token: write steps: - name: Download built archives From dd4922c227c8e1520009d1f76f5f5b8f50bbd4c5 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 23 Oct 2025 17:29:07 +0530 Subject: [PATCH 118/120] Remove API token from pypi publishing action We can not remmmove this as trusted publishing is enabled. Signed-off-by: Ayan Sinha Mahapatra --- .github/workflows/pypi-release.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 294039d1..ca12e50b 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -85,5 +85,3 @@ jobs: - name: Publish to PyPI if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} From efe5f211aa520cac8495cec650168e41178d4710 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Wed, 7 Jan 2026 14:37:15 +0530 Subject: [PATCH 119/120] Update system.py for py314 Signed-off-by: Ayan Sinha Mahapatra --- CHANGELOG.rst | 5 +++++ azure-pipelines.yml | 18 +++++++++--------- src/commoncode/system.py | 8 ++++---- tests/test_command.py | 2 -- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 4550c80e..25495220 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,11 @@ Release notes ============= +Version 32.4.1 - (2025-01-07) +----------------------------- + +- Update system.py for python3.14 + Version 32.4.0 - (2025-10-22) ----------------------------- diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1946a604..df185c43 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -25,14 +25,6 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml - parameters: - job_name: macos13_cpython - image_name: macOS-13 - python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] - test_suites: - all: venv/bin/pytest -n 2 -vvs - - template: etc/ci/azure-posix.yml parameters: job_name: macos14_cpython @@ -41,6 +33,14 @@ jobs: test_suites: all: venv/bin/pytest -n 2 -vvs + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos15_cpython + image_name: macOS-15 + python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] + test_suites: + all: venv/bin/pytest -n 2 -vvs + - template: etc/ci/azure-win.yml parameters: job_name: win2025_cpython @@ -68,7 +68,7 @@ jobs: python_versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] test_suites: click_versions: | - for clk_ver in 8.3.0 8.2.0 8.2.1 8.1.8 8.1.7 8.1.6 8.1.5 8.1.4 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7; + for clk_ver in 8.3.1 8.3.0 8.2.0 8.2.1 8.1.8 8.1.7 8.1.6 8.1.5 8.1.4 8.1.3 8.1.2 8.1.1 8.1.0 8.0.4 8.0.2 8.0.3 8.0.1 7.1.2 7.1.1 7.1 6.7; do venv/bin/pip install click==$clk_ver; venv/bin/pytest -vvs tests/test_cliutils_progressbar.py; diff --git a/src/commoncode/system.py b/src/commoncode/system.py index ea6f7d36..3c5a6744 100644 --- a/src/commoncode/system.py +++ b/src/commoncode/system.py @@ -144,11 +144,11 @@ def has_case_sensitive_fs(): py3 = _sys_v0 == 3 _sys_v1 = sys.version_info[1] -py36 = py3 and _sys_v1 == 6 -py37 = py3 and _sys_v1 == 7 -py38 = py3 and _sys_v1 == 8 -py39 = py3 and _sys_v1 == 9 py310 = py3 and _sys_v1 == 10 +py311 = py3 and _sys_v1 == 11 +py312 = py3 and _sys_v1 == 12 +py313 = py3 and _sys_v1 == 13 +py314 = py3 and _sys_v1 == 14 # Do not let Windows error pop up messages with default SetErrorMode # See http://msdn.microsoft.com/en-us/library/ms680621(VS100).aspx diff --git a/tests/test_command.py b/tests/test_command.py index 173b6c61..d0f735f1 100644 --- a/tests/test_command.py +++ b/tests/test_command.py @@ -15,14 +15,12 @@ from commoncode.system import on_linux from commoncode.system import on_mac from commoncode.system import on_windows -from commoncode.system import py36 from commoncode.testcase import FileBasedTesting class TestCommand(FileBasedTesting): test_data_dir = os.path.join(os.path.dirname(__file__), "data") - @skipIf(py36, "This fails on Python 3.6 https://bugs.python.org/issue26919") def test_execute_can_handle_non_ascii_output(self): # Popen returns a *binary* string with non-ascii chars: skips these python = sys.executable From c284fa431aef8b20dc18acb5a390345aa3e554c9 Mon Sep 17 00:00:00 2001 From: Ayan Sinha Mahapatra Date: Thu, 8 Jan 2026 13:08:50 +0530 Subject: [PATCH 120/120] Support setting the tarfile extract filter This is needed for python 3.14 support. Reference: https://github.com/aboutcode-org/commoncode/issues/88 Signed-off-by: Ayan Sinha Mahapatra --- CHANGELOG.rst | 6 ++++++ src/commoncode/archive.py | 8 ++++++-- src/commoncode/testcase.py | 8 ++++---- tests/data/filetype/types.tar | Bin 112640 -> 109056 bytes tests/test_filetype.py | 5 ++++- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 25495220..dc638663 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,12 @@ Release notes ============= +Version 32.4.2 - (2025-01-08) +----------------------------- + +- Support setting the tar archive filter in python3.14 + https://github.com/aboutcode-org/commoncode/issues/88 + Version 32.4.1 - (2025-01-07) ----------------------------- diff --git a/src/commoncode/archive.py b/src/commoncode/archive.py index 3c0c7583..1d3e42cc 100644 --- a/src/commoncode/archive.py +++ b/src/commoncode/archive.py @@ -14,6 +14,7 @@ from os import path from commoncode.system import on_windows +from commoncode.system import py314 """ Mimimal tar and zip file handling, primarily for testing. @@ -39,7 +40,7 @@ def _extract_tar_raw(test_path, target_dir, to_bytes, *args, **kwargs): extract_tar_uni = partial(_extract_tar_raw, to_bytes=False) -def extract_tar(location, target_dir, verbatim=False, *args, **kwargs): +def extract_tar(location, target_dir, verbatim=False, filter=None, *args, **kwargs): """ Extract a tar archive at location in the target_dir directory. If `verbatim` is True preserve the permissions. @@ -58,7 +59,10 @@ def extract_tar(location, target_dir, verbatim=False, *args, **kwargs): if not verbatim: tarinfo.mode = 0o755 to_extract.append(tarinfo) - tar.extractall(target_dir, members=to_extract) + if py314 and filter: + tar.extractall(target_dir, members=to_extract, filter=filter) + else: + tar.extractall(target_dir, members=to_extract) finally: if tar: tar.close() diff --git a/src/commoncode/testcase.py b/src/commoncode/testcase.py index 525bf666..2a7b37a9 100644 --- a/src/commoncode/testcase.py +++ b/src/commoncode/testcase.py @@ -194,7 +194,7 @@ def remove_vcs(self, test_dir): for tf in tilde_files: os.remove(tf) - def __extract(self, test_path, extract_func=None, verbatim=False): + def __extract(self, test_path, extract_func=None, verbatim=False, filter=None): """ Given an archive file identified by test_path relative to a test files directory, return a new temp directory where the @@ -206,7 +206,7 @@ def __extract(self, test_path, extract_func=None, verbatim=False): target_path = path.basename(test_path) target_dir = self.get_temp_dir(target_path) original_archive = self.get_test_loc(test_path) - extract_func(original_archive, target_dir, verbatim=verbatim) + extract_func(original_archive, target_dir, verbatim=verbatim, filter=filter) return target_dir def extract_test_zip(self, test_path, *args, **kwargs): @@ -215,8 +215,8 @@ def extract_test_zip(self, test_path, *args, **kwargs): def extract_test_zip_raw(self, test_path, *args, **kwargs): return self.__extract(test_path, extract_zip_raw) - def extract_test_tar(self, test_path, verbatim=False): - return self.__extract(test_path, extract_tar, verbatim) + def extract_test_tar(self, test_path, verbatim=False, filter=None): + return self.__extract(test_path, extract_tar, verbatim, filter) def extract_test_tar_raw(self, test_path, *args, **kwargs): return self.__extract(test_path, extract_tar_raw) diff --git a/tests/data/filetype/types.tar b/tests/data/filetype/types.tar index 24fc04c44da43ac26ebb53a04feca6c8d1a04342..458dc4476efa3cb3b3507135af450adc925a1bfa 100644 GIT binary patch delta 1169 zcmZqpz}9eujnhCk$kjb0GQf4>!f0^=ATTpAVE|Hw=B8lUz|h2Wa`hbAM@K!$(-P{)czX9W1*7U_n#MubdU7>3)n z57O)=hGyhjx9Jm;-(;p^E#f8&h6V=a<|byQCI*HK28M=4<|YgZJ|Iu9Lp?E3&=H%9 zH#431$H>IMFq!E@3T}sWbiIafemMbq#<)tQ!Iwd zu$h<`Gbosnmxj2^blp7N{2_jb!W}0^fYz8BFesRjXAMWNZg7BOP%tn*#Ne`IdbI6vn5dshGF`G?Hi+a{_sf&Yw#d&tOSobOQ!3LCT^r#5HYqZcJEs062jc zaHB_ny(civYeYb8QfD)1lli^xd*2tmInldQI%%Z7)k^McskLMPfLh|_ZGL5kp#-f^ zASjOkqV}4MOY=CW-t_u@mviRd`t9CT`_hlA_1*7}^ttk4#v}`xvaf{Lq|>xCT)2)z zU>pkI=Kk`r67$A%%%2Xe36Vz`S<7Gg({SWEP4tQjO)8oh8GKvB#n?QL4B5OR>Wx;S(llqQHGn_B}XdAu|n zxMiV`am>aRjJB4~s~zunSyhTEoE?