From aa03226ae461f028603e77fe97d9ab078447ffa6 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 23 Dec 2021 05:07:00 +0100 Subject: [PATCH 001/132] added asyncio timeout generator, some type hinting and stripped six --- _python_utils_tests/test_time.py | 33 +++++++++++ python_utils/converters.py | 13 ++--- python_utils/time.py | 97 +++++++++++++++++++++++--------- setup.py | 3 +- tox.ini | 4 +- 5 files changed, 114 insertions(+), 36 deletions(-) create mode 100644 _python_utils_tests/test_time.py diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py new file mode 100644 index 0000000..318f0cf --- /dev/null +++ b/_python_utils_tests/test_time.py @@ -0,0 +1,33 @@ +import pytest +import datetime +import itertools +from python_utils import time + + +@pytest.mark.asyncio +async def test_aio_timeout_generator(): + async for i in time.aio_timeout_generator(0.1, 0.06): + print(i) + + time.aio_timeout = datetime.timedelta(seconds=0.1) + interval = datetime.timedelta(seconds=0.06) + async for i in time.aio_timeout_generator(time.aio_timeout, interval, + itertools.count()): + print(i) + + async for i in time.aio_timeout_generator(1, interval=0.1, iterable='ab'): + print(i) + + # Testing small interval: + time.aio_timeout = datetime.timedelta(seconds=0.1) + interval = datetime.timedelta(seconds=0.06) + async for i in time.aio_timeout_generator(time.aio_timeout, interval, + interval_multiplier=2): + print(i) + + # Testing large interval: + time.aio_timeout = datetime.timedelta(seconds=0.1) + interval = datetime.timedelta(seconds=2) + async for i in time.aio_timeout_generator(time.aio_timeout, interval, + interval_multiplier=2): + print(i) diff --git a/python_utils/converters.py b/python_utils/converters.py index 619c104..b36bc33 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -2,7 +2,6 @@ unicode_literals) import re -import six import math import decimal @@ -68,7 +67,7 @@ def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): if regexp is True: regexp = re.compile(r'(\d+)') - elif isinstance(regexp, six.string_types): + elif isinstance(regexp, str): regexp = re.compile(regexp) elif hasattr(regexp, 'search'): pass @@ -141,7 +140,7 @@ def to_float(input_, default=0, exception=(ValueError, TypeError), if regexp is True: regexp = re.compile(r'(\d+(\.\d+|))') - elif isinstance(regexp, six.string_types): + elif isinstance(regexp, str): regexp = re.compile(regexp) elif hasattr(regexp, 'search'): pass @@ -176,10 +175,10 @@ def to_unicode(input_, encoding='utf-8', errors='replace'): >>> to_unicode(Foo) "" ''' - if isinstance(input_, six.binary_type): + if isinstance(input_, bytes): input_ = input_.decode(encoding, errors) else: - input_ = six.text_type(input_) + input_ = str(input_) return input_ @@ -200,11 +199,11 @@ def to_str(input_, encoding='utf-8', errors='replace'): >>> to_str(Foo) "" ''' - if isinstance(input_, six.binary_type): + if isinstance(input_, bytes): pass else: if not hasattr(input_, 'encode'): - input_ = six.text_type(input_) + input_ = str(input_) input_ = input_.encode(encoding, errors) return input_ diff --git a/python_utils/time.py b/python_utils/time.py index e7575d0..8735675 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -1,8 +1,21 @@ from __future__ import absolute_import -import six import time +import typing +import asyncio import datetime import itertools +import warnings + +delta_type = typing.Union[datetime.timedelta, int, float] +timestamp_type = typing.Union[ + datetime.timedelta, + datetime.date, + datetime.datetime, + str, + int, + float, + None, +] # There might be a better way to get the epoch with tzinfo, please create @@ -10,7 +23,7 @@ epoch = datetime.datetime(year=1970, month=1, day=1) -def timedelta_to_seconds(delta): +def timedelta_to_seconds(delta: datetime.timedelta): '''Convert a timedelta to seconds with the microseconds as fraction Note that this method has become largely obsolete with the @@ -36,7 +49,8 @@ def timedelta_to_seconds(delta): return total -def format_time(timestamp, precision=datetime.timedelta(seconds=1)): +def format_time(timestamp: timestamp_type, + precision: datetime.timedelta = datetime.timedelta(seconds=1)): '''Formats timedelta/datetime/seconds >>> format_time('1') @@ -61,10 +75,12 @@ def format_time(timestamp, precision=datetime.timedelta(seconds=1)): ''' precision_seconds = precision.total_seconds() - if isinstance(timestamp, six.string_types + six.integer_types + (float, )): + if isinstance(timestamp, str): + timestamp = float(timestamp) + + if isinstance(timestamp, (int, float)): try: - castfunc = six.integer_types[-1] - timestamp = datetime.timedelta(seconds=castfunc(timestamp)) + timestamp = datetime.timedelta(seconds=timestamp) except OverflowError: # pragma: no cover timestamp = None @@ -85,10 +101,7 @@ def format_time(timestamp, precision=datetime.timedelta(seconds=1)): seconds = seconds - (seconds % precision_seconds) try: # pragma: no cover - if six.PY3: - dt = datetime.datetime.fromtimestamp(seconds) - else: - dt = datetime.datetime.utcfromtimestamp(seconds) + dt = datetime.datetime.fromtimestamp(seconds) except ValueError: # pragma: no cover dt = datetime.datetime.max return str(dt) @@ -101,10 +114,10 @@ def format_time(timestamp, precision=datetime.timedelta(seconds=1)): def timeout_generator( - timeout, - interval=datetime.timedelta(seconds=1), - iterable=itertools.count, - interval_exponent=1.0, + timeout: delta_type, + interval: delta_type = datetime.timedelta(seconds=1), + iterable: typing.Iterable = itertools.count, + interval_multiplier: float = 1.0, ): ''' Generator that walks through the given iterable (a counter by default) @@ -135,7 +148,7 @@ def timeout_generator( # Testing small interval: >>> timeout = datetime.timedelta(seconds=0.1) >>> interval = datetime.timedelta(seconds=0.06) - >>> for i in timeout_generator(timeout, interval, interval_exponent=2): + >>> for i in timeout_generator(timeout, interval, interval_multiplier=2): ... print(i) 0 1 @@ -143,7 +156,7 @@ def timeout_generator( # Testing large interval: >>> timeout = datetime.timedelta(seconds=0.1) >>> interval = datetime.timedelta(seconds=2) - >>> for i in timeout_generator(timeout, interval, interval_exponent=2): + >>> for i in timeout_generator(timeout, interval, interval_multiplier=2): ... print(i) 0 1 @@ -158,20 +171,52 @@ def timeout_generator( if callable(iterable): iterable = iterable() - if interval < 1: - interval_exponent = 1.0 / interval_exponent + end = timeout + time.perf_counter() + for item in iterable: + yield item + + if time.perf_counter() >= end: + break - if six.PY3: # pragma: no cover - timer = time.perf_counter - else: - timer = time.time + interval *= interval_multiplier + time.sleep(interval) - end = timeout + timer() + +async def aio_timeout_generator( + timeout: delta_type, + interval: delta_type = datetime.timedelta(seconds=1), + iterable: typing.Iterable = itertools.count, + interval_multiplier: float = 1.0, +): + ''' + Aync generator that walks through the given iterable (a counter by default) + until the timeout is reached with a configurable interval between items + + The interval_exponent automatically increases the timeout with each run. + Note that if the interval is less than 1, 1/interval_exponent will be used + so the interval is always growing. To double the interval with each run, + specify 2. + + Doctests and asyncio are not friends, so no examples. But this function is + effectively the same as the timeout_generor but it uses `async for` + instead. + ''' + + if isinstance(interval, datetime.timedelta): + interval: int = timedelta_to_seconds(interval) + + if isinstance(timeout, datetime.timedelta): + timeout = timedelta_to_seconds(timeout) + + if callable(iterable): + iterable = iterable() + + end = timeout + time.perf_counter() for item in iterable: yield item - if timer() >= end: + if time.perf_counter() >= end: break - interval **= interval_exponent - time.sleep(interval) + await asyncio.sleep(interval) + interval *= interval_multiplier diff --git a/setup.py b/setup.py index f3f080a..547ace8 100644 --- a/setup.py +++ b/setup.py @@ -31,8 +31,8 @@ packages=setuptools.find_packages(exclude=[ '_python_utils_tests', '*.__pycache__']), long_description=long_description, - install_requires=['six'], tests_require=['pytest'], + python_requires='>3.5', extras_require={ 'docs': [ 'mock', @@ -44,6 +44,7 @@ 'pytest', 'pytest-cov', 'pytest-flake8', + 'pytest-asyncio', 'sphinx', ], }, diff --git a/tox.ini b/tox.ini index 7f974b1..d9168e0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,15 @@ [tox] -envlist = py27, py35, py36, py37, py38, py39, pypy, flake8, docs +envlist = py35, py36, py37, py38, py39, py310, pypy3, flake8, docs skip_missing_interpreters = True [testenv] basepython = - py27: python2.7 py35: python3.5 py36: python3.6 py37: python3.7 py38: python3.8 py39: python3.9 + py310: python3.10 pypy: pypy setenv = PY_IGNORE_IMPORTMISMATCH=1 From adb8bfbb09691880844fd7b5917ab315a59c7c1d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 31 Dec 2021 13:51:43 +0100 Subject: [PATCH 002/132] added maximum interval with tests --- _python_utils_tests/test_import.py | 2 -- _python_utils_tests/test_time.py | 43 +++++++++++------------------- python_utils/time.py | 15 +++++------ 3 files changed, 22 insertions(+), 38 deletions(-) diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index e9f9255..d699e20 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -10,8 +10,6 @@ def relative_import(level): locals_ = {} globals_ = {'__name__': 'python_utils.import_'} import_.import_global('.formatters', locals_=locals_, globals_=globals_) - import pprint - pprint.pprint(globals_) assert 'camel_to_underscore' in globals_ diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 318f0cf..fc92ab1 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -1,33 +1,22 @@ -import pytest import datetime import itertools -from python_utils import time - - -@pytest.mark.asyncio -async def test_aio_timeout_generator(): - async for i in time.aio_timeout_generator(0.1, 0.06): - print(i) - time.aio_timeout = datetime.timedelta(seconds=0.1) - interval = datetime.timedelta(seconds=0.06) - async for i in time.aio_timeout_generator(time.aio_timeout, interval, - itertools.count()): - print(i) +import pytest - async for i in time.aio_timeout_generator(1, interval=0.1, iterable='ab'): - print(i) +from python_utils import time - # Testing small interval: - time.aio_timeout = datetime.timedelta(seconds=0.1) - interval = datetime.timedelta(seconds=0.06) - async for i in time.aio_timeout_generator(time.aio_timeout, interval, - interval_multiplier=2): - print(i) - # Testing large interval: - time.aio_timeout = datetime.timedelta(seconds=0.1) - interval = datetime.timedelta(seconds=2) - async for i in time.aio_timeout_generator(time.aio_timeout, interval, - interval_multiplier=2): - print(i) +@pytest.mark.parametrize('timeout', [0.01, datetime.timedelta(seconds=0.01)]) +@pytest.mark.parametrize('interval', [0.006, datetime.timedelta(seconds=0.006)]) +@pytest.mark.parametrize('interval_multiplier', [0.5, 1.0, 2.0]) +@pytest.mark.parametrize('maximum_interval', [0.01, datetime.timedelta( + seconds=0.01), None]) +@pytest.mark.parametrize('iterable', ['ab', itertools.count(), itertools.count]) +@pytest.mark.asyncio +async def test_aio_timeout_generator(iterable, timeout, interval, + interval_multiplier, + maximum_interval): + async for _ in time.aio_timeout_generator( + timeout, interval, iterable, + maximum_interval=maximum_interval): + pass diff --git a/python_utils/time.py b/python_utils/time.py index 8735675..164b6b6 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -145,21 +145,12 @@ def timeout_generator( a b - # Testing small interval: >>> timeout = datetime.timedelta(seconds=0.1) >>> interval = datetime.timedelta(seconds=0.06) >>> for i in timeout_generator(timeout, interval, interval_multiplier=2): ... print(i) 0 1 - - # Testing large interval: - >>> timeout = datetime.timedelta(seconds=0.1) - >>> interval = datetime.timedelta(seconds=2) - >>> for i in timeout_generator(timeout, interval, interval_multiplier=2): - ... print(i) - 0 - 1 ''' if isinstance(interval, datetime.timedelta): @@ -187,6 +178,7 @@ async def aio_timeout_generator( interval: delta_type = datetime.timedelta(seconds=1), iterable: typing.Iterable = itertools.count, interval_multiplier: float = 1.0, + maximum_interval: delta_type = None, ): ''' Aync generator that walks through the given iterable (a counter by default) @@ -205,6 +197,9 @@ async def aio_timeout_generator( if isinstance(interval, datetime.timedelta): interval: int = timedelta_to_seconds(interval) + if isinstance(maximum_interval, datetime.timedelta): + maximum_interval: int = timedelta_to_seconds(maximum_interval) + if isinstance(timeout, datetime.timedelta): timeout = timedelta_to_seconds(timeout) @@ -220,3 +215,5 @@ async def aio_timeout_generator( await asyncio.sleep(interval) interval *= interval_multiplier + if maximum_interval: + interval = min(interval, maximum_interval) From 7f777c967961fa5b211c4e7910b0e2024531c3e7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 13:48:23 +0100 Subject: [PATCH 003/132] Update documentation and setup.py formats --- docs/Makefile | 159 +-------- docs/_theme/LICENSE | 46 --- docs/_theme/flask_theme_support.py | 86 ----- docs/_theme/wolph/layout.html | 16 - docs/_theme/wolph/relations.html | 19 - docs/_theme/wolph/static/flasky.css_t | 431 ----------------------- docs/_theme/wolph/static/small_flask.css | 70 ---- docs/_theme/wolph/theme.conf | 7 - docs/conf.py | 301 ++-------------- docs/index.rst | 16 +- docs/make.bat | 189 +--------- setup.py | 16 +- 12 files changed, 70 insertions(+), 1286 deletions(-) delete mode 100644 docs/_theme/LICENSE delete mode 100644 docs/_theme/flask_theme_support.py delete mode 100644 docs/_theme/wolph/layout.html delete mode 100644 docs/_theme/wolph/relations.html delete mode 100644 docs/_theme/wolph/static/flasky.css_t delete mode 100644 docs/_theme/wolph/static/small_flask.css delete mode 100644 docs/_theme/wolph/theme.conf diff --git a/docs/Makefile b/docs/Makefile index 7cab5b5..d4bb2cb 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,153 +1,20 @@ -# Makefile for Sphinx documentation +# Minimal makefile for Sphinx documentation # -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . BUILDDIR = _build -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - +# Put it first so that "make" without argument is like "make help". help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PythonUtils.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PythonUtils.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/PythonUtils" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PythonUtils" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." +.PHONY: help Makefile -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_theme/LICENSE b/docs/_theme/LICENSE deleted file mode 100644 index f258ba0..0000000 --- a/docs/_theme/LICENSE +++ /dev/null @@ -1,46 +0,0 @@ -Modifications: - -Copyright (c) 2012 Rick van Hattem. - - -Original Projects: - -Copyright (c) 2010 Kenneth Reitz. -Copyright (c) 2010 by Armin Ronacher. - - -Some rights reserved. - -Redistribution and use in source and binary forms of the theme, with or -without modification, are permitted provided that the following conditions -are met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -* The names of the contributors may not be used to endorse or - promote products derived from this software without specific - prior written permission. - -We kindly ask you to only use these themes in an unmodified manner just -for Flask and Flask-related products, not for unrelated projects. If you -like the visual style and want to use it for your own projects, please -consider making some larger changes to the themes (such as changing -font faces, sizes, colors or margins). - -THIS THEME IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS THEME, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. diff --git a/docs/_theme/flask_theme_support.py b/docs/_theme/flask_theme_support.py deleted file mode 100644 index 33f4744..0000000 --- a/docs/_theme/flask_theme_support.py +++ /dev/null @@ -1,86 +0,0 @@ -# flasky extensions. flasky pygments style based on tango style -from pygments.style import Style -from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic, Whitespace, Punctuation, Other, Literal - - -class FlaskyStyle(Style): - background_color = "#f8f8f8" - default_style = "" - - styles = { - # No corresponding class for the following: - #Text: "", # class: '' - Whitespace: "underline #f8f8f8", # class: 'w' - Error: "#a40000 border:#ef2929", # class: 'err' - Other: "#000000", # class 'x' - - Comment: "italic #8f5902", # class: 'c' - Comment.Preproc: "noitalic", # class: 'cp' - - Keyword: "bold #004461", # class: 'k' - Keyword.Constant: "bold #004461", # class: 'kc' - Keyword.Declaration: "bold #004461", # class: 'kd' - Keyword.Namespace: "bold #004461", # class: 'kn' - Keyword.Pseudo: "bold #004461", # class: 'kp' - Keyword.Reserved: "bold #004461", # class: 'kr' - Keyword.Type: "bold #004461", # class: 'kt' - - Operator: "#582800", # class: 'o' - Operator.Word: "bold #004461", # class: 'ow' - like keywords - - Punctuation: "bold #000000", # class: 'p' - - # because special names such as Name.Class, Name.Function, etc. - # are not recognized as such later in the parsing, we choose them - # to look the same as ordinary variables. - Name: "#000000", # class: 'n' - Name.Attribute: "#c4a000", # class: 'na' - to be revised - Name.Builtin: "#004461", # class: 'nb' - Name.Builtin.Pseudo: "#3465a4", # class: 'bp' - Name.Class: "#000000", # class: 'nc' - to be revised - Name.Constant: "#000000", # class: 'no' - to be revised - Name.Decorator: "#888", # class: 'nd' - to be revised - Name.Entity: "#ce5c00", # class: 'ni' - Name.Exception: "bold #cc0000", # class: 'ne' - Name.Function: "#000000", # class: 'nf' - Name.Property: "#000000", # class: 'py' - Name.Label: "#f57900", # class: 'nl' - Name.Namespace: "#000000", # class: 'nn' - to be revised - Name.Other: "#000000", # class: 'nx' - Name.Tag: "bold #004461", # class: 'nt' - like a keyword - Name.Variable: "#000000", # class: 'nv' - to be revised - Name.Variable.Class: "#000000", # class: 'vc' - to be revised - Name.Variable.Global: "#000000", # class: 'vg' - to be revised - Name.Variable.Instance: "#000000", # class: 'vi' - to be revised - - Number: "#990000", # class: 'm' - - Literal: "#000000", # class: 'l' - Literal.Date: "#000000", # class: 'ld' - - String: "#4e9a06", # class: 's' - String.Backtick: "#4e9a06", # class: 'sb' - String.Char: "#4e9a06", # class: 'sc' - String.Doc: "italic #8f5902", # class: 'sd' - like a comment - String.Double: "#4e9a06", # class: 's2' - String.Escape: "#4e9a06", # class: 'se' - String.Heredoc: "#4e9a06", # class: 'sh' - String.Interpol: "#4e9a06", # class: 'si' - String.Other: "#4e9a06", # class: 'sx' - String.Regex: "#4e9a06", # class: 'sr' - String.Single: "#4e9a06", # class: 's1' - String.Symbol: "#4e9a06", # class: 'ss' - - Generic: "#000000", # class: 'g' - Generic.Deleted: "#a40000", # class: 'gd' - Generic.Emph: "italic #000000", # class: 'ge' - Generic.Error: "#ef2929", # class: 'gr' - Generic.Heading: "bold #000080", # class: 'gh' - Generic.Inserted: "#00A000", # class: 'gi' - Generic.Output: "#888", # class: 'go' - Generic.Prompt: "#745334", # class: 'gp' - Generic.Strong: "bold #000000", # class: 'gs' - Generic.Subheading: "bold #800080", # class: 'gu' - Generic.Traceback: "bold #a40000", # class: 'gt' - } diff --git a/docs/_theme/wolph/layout.html b/docs/_theme/wolph/layout.html deleted file mode 100644 index a39162f..0000000 --- a/docs/_theme/wolph/layout.html +++ /dev/null @@ -1,16 +0,0 @@ -{%- extends "basic/layout.html" %} -{%- block extrahead %} - {{ super() }} - {% if theme_touch_icon %} - - {% endif %} - -{% endblock %} -{%- block relbar2 %}{% endblock %} -{%- block footer %} -
- © Copyright {{ copyright }}. - Created using Sphinx. -
-{%- endblock %} diff --git a/docs/_theme/wolph/relations.html b/docs/_theme/wolph/relations.html deleted file mode 100644 index 3bbcde8..0000000 --- a/docs/_theme/wolph/relations.html +++ /dev/null @@ -1,19 +0,0 @@ -

Related Topics

- diff --git a/docs/_theme/wolph/static/flasky.css_t b/docs/_theme/wolph/static/flasky.css_t deleted file mode 100644 index 71aae28..0000000 --- a/docs/_theme/wolph/static/flasky.css_t +++ /dev/null @@ -1,431 +0,0 @@ -/* - * flasky.css_t - * ~~~~~~~~~~~~ - * - * :copyright: Copyright 2010 by Armin Ronacher. Modifications by Kenneth Reitz. - * :license: Flask Design License, see LICENSE for details. - */ - -{% set page_width = '940px' %} -{% set sidebar_width = '220px' %} - -@import url("basic.css"); - -/* -- page layout ----------------------------------------------------------- */ - -body { - font-family: 'goudy old style', 'minion pro', 'bell mt', Georgia, 'Hiragino Mincho Pro'; - font-size: 17px; - background-color: white; - color: #000; - margin: 0; - padding: 0; -} - -div.document { - width: {{ page_width }}; - margin: 30px auto 0 auto; -} - -div.documentwrapper { - float: left; - width: 100%; -} - -div.bodywrapper { - margin: 0 0 0 {{ sidebar_width }}; -} - -div.sphinxsidebar { - width: {{ sidebar_width }}; -} - -hr { - border: 1px solid #B1B4B6; -} - -div.body { - background-color: #ffffff; - color: #3E4349; - padding: 0 30px 0 30px; -} - -img.floatingflask { - padding: 0 0 10px 10px; - float: right; -} - -div.footer { - width: {{ page_width }}; - margin: 20px auto 30px auto; - font-size: 14px; - color: #888; - text-align: right; -} - -div.footer a { - color: #888; -} - -div.related { - display: none; -} - -div.sphinxsidebar a { - color: #444; - text-decoration: none; - border-bottom: 1px dotted #999; -} - -div.sphinxsidebar a:hover { - border-bottom: 1px solid #999; -} - -div.sphinxsidebar { - font-size: 14px; - line-height: 1.5; -} - -div.sphinxsidebarwrapper { - padding: 0px 10px; -} - -div.sphinxsidebarwrapper p.logo { - padding: 0 0 20px 0; - margin: 0; - text-align: center; -} - -div.sphinxsidebar h3, -div.sphinxsidebar h4 { - font-family: 'Garamond', 'Georgia', serif; - color: #555; - font-size: 24px; - font-weight: normal; - margin: 0 0 5px 0; - padding: 0; -} - -div.sphinxsidebar h4 { - font-size: 20px; -} - -div.sphinxsidebar h3 a { - color: #444; -} - -div.sphinxsidebar p.logo a, -div.sphinxsidebar h3 a, -div.sphinxsidebar p.logo a:hover, -div.sphinxsidebar h3 a:hover { - border: none; -} - -div.sphinxsidebar p { - color: #555; - margin: 10px 0; -} - -div.sphinxsidebar ul { - margin: 10px 0; - padding: 0; - color: #000; -} - -div.sphinxsidebar input[type="text"] { - width: 160px!important; -} -div.sphinxsidebar input { - border: 1px solid #ccc; - font-family: 'Georgia', serif; - font-size: 1em; -} - -/* -- body styles ----------------------------------------------------------- */ - -a { - color: #004B6B; - text-decoration: underline; -} - -a:hover { - color: #6D4100; - text-decoration: underline; -} - -div.body h1, -div.body h2, -div.body h3, -div.body h4, -div.body h5, -div.body h6 { - font-family: 'Garamond', 'Georgia', serif; - font-weight: normal; - margin: 30px 0px 10px 0px; - padding: 0; -} - -div.body h1 { margin-top: 0; padding-top: 0; font-size: 240%; } -div.body h2 { font-size: 180%; } -div.body h3 { font-size: 150%; } -div.body h4 { font-size: 130%; } -div.body h5 { font-size: 100%; } -div.body h6 { font-size: 100%; } - -a.headerlink { - color: #ddd; - padding: 0 4px; - text-decoration: none; -} - -a.headerlink:hover { - color: #444; - background: #eaeaea; -} - -div.body p, div.body dd, div.body li { - line-height: 1.4em; -} - -div.admonition { - background: #fafafa; - margin: 20px -30px; - padding: 10px 30px; - border-top: 1px solid #ccc; - border-bottom: 1px solid #ccc; -} - -div.admonition tt.xref, div.admonition a tt { - border-bottom: 1px solid #fafafa; -} - -dd div.admonition { - margin-left: -60px; - padding-left: 60px; -} - -div.admonition p.admonition-title { - font-family: 'Garamond', 'Georgia', serif; - font-weight: normal; - font-size: 24px; - margin: 0 0 10px 0; - padding: 0; - line-height: 1; -} - -div.admonition p.last { - margin-bottom: 0; -} - -div.highlight { - background-color: white; -} - -dt:target, .highlight { - background: #FAF3E8; -} - -div.note { - background-color: #eee; - border: 1px solid #ccc; -} - -div.seealso { - background-color: #ffc; - border: 1px solid #ff6; -} - -div.topic { - background-color: #eee; -} - -p.admonition-title { - display: inline; -} - -p.admonition-title:after { - content: ":"; -} - -pre, tt { - font-family: 'Consolas', 'Menlo', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace; - font-size: 0.9em; -} - -img.screenshot { -} - -tt.descname, tt.descclassname { - font-size: 0.95em; -} - -tt.descname { - padding-right: 0.08em; -} - -img.screenshot { - -moz-box-shadow: 2px 2px 4px #eee; - -webkit-box-shadow: 2px 2px 4px #eee; - box-shadow: 2px 2px 4px #eee; -} - -table.docutils { - border: 1px solid #888; - -moz-box-shadow: 2px 2px 4px #eee; - -webkit-box-shadow: 2px 2px 4px #eee; - box-shadow: 2px 2px 4px #eee; -} - -table.docutils td, table.docutils th { - border: 1px solid #888; - padding: 0.25em 0.7em; -} - -table.field-list, table.footnote { - border: none; - -moz-box-shadow: none; - -webkit-box-shadow: none; - box-shadow: none; -} - -table.footnote { - margin: 15px 0; - width: 100%; - border: 1px solid #eee; - background: #fdfdfd; - font-size: 0.9em; -} - -table.footnote + table.footnote { - margin-top: -15px; - border-top: none; -} - -table.field-list th { - padding: 0 0.8em 0 0; -} - -table.field-list td { - padding: 0; -} - -table.footnote td.label { - width: 0px; - padding: 0.3em 0 0.3em 0.5em; -} - -table.footnote td { - padding: 0.3em 0.5em; -} - -dl { - margin: 0; - padding: 0; -} - -dl dd { - margin-left: 30px; -} - -blockquote { - margin: 0 0 0 30px; - padding: 0; -} - -ul, ol { - margin: 10px 0 10px 30px; - padding: 0; -} - -pre { - background: #eee; - padding: 7px 30px; - margin: 15px -30px; - line-height: 1.3em; -} - -dl pre, blockquote pre, li pre { - margin-left: -60px; - padding-left: 60px; -} - -dl dl pre { - margin-left: -90px; - padding-left: 90px; -} - -tt { - background-color: #ecf0f3; - color: #222; - /* padding: 1px 2px; */ -} - -tt.xref, a tt { - background-color: #FBFBFB; - border-bottom: 1px solid white; -} - -a.reference { - text-decoration: none; - border-bottom: 1px dotted #004B6B; -} - -a.reference:hover { - border-bottom: 1px solid #6D4100; -} - -a.footnote-reference { - text-decoration: none; - font-size: 0.7em; - vertical-align: top; - border-bottom: 1px dotted #004B6B; -} - -a.footnote-reference:hover { - border-bottom: 1px solid #6D4100; -} - -a:hover tt { - background: #EEE; -} - - -/* scrollbars */ - -::-webkit-scrollbar { - width: 6px; - height: 6px; -} - -::-webkit-scrollbar-button:start:decrement, -::-webkit-scrollbar-button:end:increment { - display: block; - height: 10px; -} - -::-webkit-scrollbar-button:vertical:increment { - background-color: #fff; -} - -::-webkit-scrollbar-track-piece { - background-color: #eee; - -webkit-border-radius: 3px; -} - -::-webkit-scrollbar-thumb:vertical { - height: 50px; - background-color: #ccc; - -webkit-border-radius: 3px; -} - -::-webkit-scrollbar-thumb:horizontal { - width: 50px; - background-color: #ccc; - -webkit-border-radius: 3px; -} - -/* misc. */ - -.revsys-inline { - display: none!important; -} diff --git a/docs/_theme/wolph/static/small_flask.css b/docs/_theme/wolph/static/small_flask.css deleted file mode 100644 index 1c6df30..0000000 --- a/docs/_theme/wolph/static/small_flask.css +++ /dev/null @@ -1,70 +0,0 @@ -/* - * small_flask.css_t - * ~~~~~~~~~~~~~~~~~ - * - * :copyright: Copyright 2010 by Armin Ronacher. - * :license: Flask Design License, see LICENSE for details. - */ - -body { - margin: 0; - padding: 20px 30px; -} - -div.documentwrapper { - float: none; - background: white; -} - -div.sphinxsidebar { - display: block; - float: none; - width: 102.5%; - margin: 50px -30px -20px -30px; - padding: 10px 20px; - background: #333; - color: white; -} - -div.sphinxsidebar h3, div.sphinxsidebar h4, div.sphinxsidebar p, -div.sphinxsidebar h3 a { - color: white; -} - -div.sphinxsidebar a { - color: #aaa; -} - -div.sphinxsidebar p.logo { - display: none; -} - -div.document { - width: 100%; - margin: 0; -} - -div.related { - display: block; - margin: 0; - padding: 10px 0 20px 0; -} - -div.related ul, -div.related ul li { - margin: 0; - padding: 0; -} - -div.footer { - display: none; -} - -div.bodywrapper { - margin: 0; -} - -div.body { - min-height: 0; - padding: 0; -} diff --git a/docs/_theme/wolph/theme.conf b/docs/_theme/wolph/theme.conf deleted file mode 100644 index 307a1f0..0000000 --- a/docs/_theme/wolph/theme.conf +++ /dev/null @@ -1,7 +0,0 @@ -[theme] -inherit = basic -stylesheet = flasky.css -pygments_style = flask_theme_support.FlaskyStyle - -[options] -touch_icon = diff --git a/docs/conf.py b/docs/conf.py index 2b2536d..3dacd3d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,33 +1,37 @@ -# -*- coding: utf-8 -*- +# Configuration file for the Sphinx documentation builder. # -# Python Utils documentation build configuration file, created by -# sphinx-quickstart on Wed May 9 16:57:31 2012. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html -import os -import sys -import datetime +# -- Path setup -------------------------------------------------------------- # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. +# +from datetime import date +import os +import sys sys.path.insert(0, os.path.abspath('..')) + from python_utils import __about__ -# -- General configuration ----------------------------------------------------- +# -- Project information ----------------------------------------------------- + +project = 'Python Utils' +author = __about__.__author__ +copyright = f'{date.today().year}, {author}' + +# The full version, including alpha/beta/rc tags +release = __about__.__version__ -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', @@ -40,269 +44,20 @@ # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = u'Python Utils' -copyright = u'%s, %s' % ( - datetime.date.today().year, - __about__.__author__, -) - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = __about__.__version__ -# The full version, including alpha/beta/rc tags. -release = __about__.__version__ - -suppress_warnings = [ - 'image.nonlocal_uri', -] - -needs_sphinx = '1.4' - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- +# -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'wolph' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -html_theme_path = ['_theme'] - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None +# +html_theme = 'alabaster' # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'PythonUtilsdoc' - - -# -- Options for LaTeX output -------------------------------------------------- - -latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'PythonUtils.tex', u'Python Utils Documentation', - __about__.__author__, 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'pythonutils', u'Python Utils Documentation', - [__about__.__author__], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------------ - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'PythonUtils', u'Python Utils Documentation', - __about__.__author__, 'PythonUtils', __about__.__description__, - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - - -# -- Options for Epub output --------------------------------------------------- - -# Bibliographic Dublin Core info. -epub_title = u'Python Utils' -epub_author = __about__.__author__ -epub_publisher = __about__.__author__ -epub_copyright = copyright - -# The language of the text. It defaults to the language option -# or en if the language is not set. -#epub_language = '' - -# The scheme of the identifier. Typical schemes are ISBN or URL. -#epub_scheme = '' - -# The unique identifier of the text. This can be a ISBN number -# or the project homepage. -#epub_identifier = '' - -# A unique identification for the text. -#epub_uid = '' - -# A tuple containing the cover image and cover page html template filenames. -#epub_cover = () - -# HTML files that should be inserted before the pages created by sphinx. -# The format is a list of tuples containing the path and title. -#epub_pre_files = [] - -# HTML files shat should be inserted after the pages created by sphinx. -# The format is a list of tuples containing the path and title. -#epub_post_files = [] - -# A list of files that should not be packed into the epub file. -#epub_exclude_files = [] - -# The depth of the table of contents in toc.ncx. -#epub_tocdepth = 3 - -# Allow duplicate toc entries. -#epub_tocdup = True - - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/docs/index.rst b/docs/index.rst index 3715735..df4d31a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,6 +1,12 @@ Welcome to Python Utils's documentation! ======================================== +.. image:: https://github.com/WoLpH/python-utils/actions/workflows/main.yml/badge.svg?branch=master + :target: https://github.com/WoLpH/python-utils/actions/workflows/main.yml/badge.svg?branch=master + +.. image:: https://coveralls.io/repos/WoLpH/python-utils/badge.svg?branch=master + :target: https://coveralls.io/r/WoLpH/python-utils?branch=master + Contents: .. toctree:: @@ -9,16 +15,6 @@ Contents: usage python_utils -Travis status: - -.. image:: https://travis-ci.org/WoLpH/python-utils.png?branch=master - :target: https://travis-ci.org/WoLpH/python-utils - -Coverage: - -.. image:: https://coveralls.io/repos/WoLpH/python-utils/badge.png?branch=master - :target: https://coveralls.io/r/WoLpH/python-utils?branch=master - Indices and tables ================== diff --git a/docs/make.bat b/docs/make.bat index 3342faf..8084272 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -1,190 +1,35 @@ @ECHO OFF +pushd %~dp0 + REM Command file for Sphinx documentation if "%SPHINXBUILD%" == "" ( set SPHINXBUILD=sphinx-build ) +set SOURCEDIR=. set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) if "%1" == "" goto help -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. echo. - echo.Build finished; now you can process the JSON files. - goto end + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 ) -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\PythonUtils.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\PythonUtils.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% :end +popd diff --git a/setup.py b/setup.py index 61d17a0..6910c18 100644 --- a/setup.py +++ b/setup.py @@ -1,26 +1,22 @@ import os -import sys +import typing + import setuptools # To prevent importing about and thereby breaking the coverage info we use this # exec hack -about = {} +about: typing.Dict[str, str] = {} with open('python_utils/__about__.py') as fp: exec(fp.read(), about) - if os.path.isfile('README.rst'): long_description = open('README.rst').read() else: long_description = 'See http://pypi.python.org/pypi/python-utils/' - -needs_pytest = set(['ptr', 'pytest', 'test']).intersection(sys.argv) -pytest_runner = ['pytest-runner'] if needs_pytest else [] - - if __name__ == '__main__': setuptools.setup( + python_requires='>3.6.0', name='python-utils', version=about['__version__'], author=about['__author__'], @@ -44,12 +40,12 @@ 'flake8', 'pytest', 'pytest-cov', + 'pytest-mypy', 'pytest-flake8', 'pytest-asyncio', 'sphinx', + 'types-setuptools', ], }, - setup_requires=[] + pytest_runner, classifiers=['License :: OSI Approved :: BSD License'], ) - From 276e1d2c9f81da2fa4bfbf3e3c25e0c133dfc505 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 13:50:13 +0100 Subject: [PATCH 004/132] Added full type hinting with tests --- README.rst | 69 ++++++++++++++++- _python_utils_tests/test_time.py | 52 +++++++++++-- pytest.ini | 1 + python_utils/__about__.py | 13 ++-- python_utils/converters.py | 69 ++++++++++++----- python_utils/formatters.py | 7 +- python_utils/import_.py | 37 +++++---- python_utils/logger.py | 22 +++--- python_utils/terminal.py | 17 +++-- python_utils/time.py | 127 ++++++++++++++++++++----------- setup.cfg | 4 + 11 files changed, 304 insertions(+), 114 deletions(-) diff --git a/README.rst b/README.rst index f59d94c..6018a27 100644 --- a/README.rst +++ b/README.rst @@ -58,6 +58,31 @@ format. Examples ------------------------------------------------------------------------------ +Automatically converting a generator to a list, dict or other collections +using a decorator: + +.. code-block:: pycon + + >>> @decorators.listify() + ... def generate_list(): + ... yield 1 + ... yield 2 + ... yield 3 + ... + >>> generate_list() + [1, 2, 3] + + >>> @listify(collection=dict) + ... def dict_generator(): + ... yield 'a', 1 + ... yield 'b', 2 + + >>> dict_generator() + {'a': 1, 'b': 2} + +Retrying until timeout +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + To easily retry a block of code with a configurable timeout, you can use the `time.timeout_generator`: @@ -69,6 +94,9 @@ To easily retry a block of code with a configurable timeout, you can use the ... except Exception as e: ... # Handle the exception +Formatting of timestamps, dates and times +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Easy formatting of timestamps and calculating the time since: .. code-block:: pycon @@ -98,12 +126,15 @@ Easy formatting of timestamps and calculating the time since: '1 minute ago' Converting your test from camel-case to underscores: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: pycon >>> camel_to_underscore('SpamEggsAndBacon') 'spam_eggs_and_bacon' +Attribute setting decorator. Very useful for the Django admin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A convenient decorator to set function attributes using a decorator: .. code-block:: pycon @@ -119,7 +150,11 @@ A convenient decorator to set function attributes using a decorator: >>> upper_case_name.short_description = 'Name' -Or to scale numbers: +This can be very useful for the Django admin as it allows you to have all +metadata in one place. + +Scaling numbers between ranges +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: pycon @@ -130,7 +165,8 @@ Or to scale numbers: >>> remap(decimal.Decimal('250.0'), 0.0, 1000.0, 0.0, 100.0) Decimal('25.0') -To get the screen/window/terminal size in characters: +Get the screen/window/terminal size in characters: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: pycon @@ -140,7 +176,8 @@ To get the screen/window/terminal size in characters: That method supports IPython and Jupyter as well as regular shells, using `blessings` and other modules depending on what is available. -To extract a number from nearly every string: +Extracting numbers from nearly every string: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. code-block:: pycon @@ -151,6 +188,9 @@ To extract a number from nearly every string: >>> number = converters.to_int('spam', default=1) 1 +Doing a global import of all the modules in a package programmatically: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + To do a global import programmatically you can use the `import_global` function. This effectively emulates a `from ... import *` @@ -161,6 +201,9 @@ function. This effectively emulates a `from ... import *` # The following is the equivalent of `from some_module import *` import_global('some_module') +Automatically named logger for classes: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + Or add a correclty named logger to your classes which can be easily accessed: .. code-block:: python @@ -183,3 +226,23 @@ Or add a correclty named logger to your classes which can be easily accessed: import logging my_class.log(logging.ERROR, 'log') + +Convenient type aliases and some commonly used types: + +.. code-block:: python + + # For type hinting scopes such as locals/globals/vars + Scope = Dict[str, Any] + OptionalScope = O[Scope] + + # Note that Number is only useful for extra clarity since float + # will work for both int and float in practice. + Number = U[int, float] + DecimalNumber = U[Number, decimal.Decimal] + + # To accept an exception or list of exceptions + ExceptionType = Type[Exception] + ExceptionsType = U[Tuple[ExceptionType, ...], ExceptionType] + + # Matching string/bytes types: + StringTypes = U[str, bytes] diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index fc92ab1..a6e9f4a 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -3,20 +3,56 @@ import pytest +from python_utils import aio from python_utils import time -@pytest.mark.parametrize('timeout', [0.01, datetime.timedelta(seconds=0.01)]) -@pytest.mark.parametrize('interval', [0.006, datetime.timedelta(seconds=0.006)]) -@pytest.mark.parametrize('interval_multiplier', [0.5, 1.0, 2.0]) -@pytest.mark.parametrize('maximum_interval', [0.01, datetime.timedelta( - seconds=0.01), None]) -@pytest.mark.parametrize('iterable', ['ab', itertools.count(), itertools.count]) +@pytest.mark.parametrize( + 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ + (0.01, 0.006, 0.5, 0.01, aio.acount, 2), + (0.01, 0.006, 0.5, 0.01, aio.acount, 2), + (0.01, 0.006, 0.5, 0.01, aio.acount(), 2), + (0.01, 0.006, 1.0, None, aio.acount, 2), + (datetime.timedelta(seconds=0.01), + datetime.timedelta(seconds=0.006), + 2.0, datetime.timedelta(seconds=0.01), aio.acount, + 2), + ]) @pytest.mark.asyncio async def test_aio_timeout_generator(iterable, timeout, interval, interval_multiplier, - maximum_interval): - async for _ in time.aio_timeout_generator( + maximum_interval, result): + i = None + async for i in time.aio_timeout_generator( timeout, interval, iterable, maximum_interval=maximum_interval): pass + + assert i == result + + +@pytest.mark.parametrize( + 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ + (0.01, 0.006, 0.5, 0.01, 'abc', 'c'), + (0.01, 0.006, 0.5, 0.01, itertools.count, 2), + (0.01, 0.006, 0.5, 0.01, itertools.count(), 2), + (0.01, 0.006, 1.0, None, 'abc', 'c'), + (datetime.timedelta(seconds=0.01), + datetime.timedelta(seconds=0.006), + 2.0, datetime.timedelta(seconds=0.01), + itertools.count, 2), + ]) +def test_timeout_generator( + iterable, timeout, interval, interval_multiplier, + maximum_interval, result): + i = None + for i in time.timeout_generator( + timeout=timeout, + interval=interval, + interval_multiplier=interval_multiplier, + iterable=iterable, + maximum_interval=maximum_interval, + ): + pass + + assert i == result diff --git a/pytest.ini b/pytest.ini index 2de1634..b43425d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,6 +8,7 @@ addopts = --cov python_utils --cov-report html --cov-report term-missing + --mypy flake8-ignore = *.py W391 diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 24c2cb7..4a55392 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,9 +1,8 @@ -__package_name__ = 'python-utils' -__version__ = '2.7.1' -__author__ = 'Rick van Hattem' -__author_email__ = 'Wolph@wol.ph' -__description__ = ( +__package_name__: str = 'python-utils' +__version__: str = '2.7.1' +__author__: str = 'Rick van Hattem' +__author_email__: str = 'Wolph@wol.ph' +__description__: str = ( 'Python Utils is a module with some convenient utilities not included ' 'with the standard Python install') -__url__ = 'https://github.com/WoLpH/python-utils' - +__url__: str = 'https://github.com/WoLpH/python-utils' diff --git a/python_utils/converters.py b/python_utils/converters.py index 0a9517f..fb6800f 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -6,11 +6,17 @@ import decimal import math import re -import math -import decimal +import typing + +from . import types -def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): +def to_int( + input_: typing.Optional[str] = None, + default: int = 0, + exception: types.ExceptionsType = (ValueError, TypeError), + regexp: types.O[re.Pattern] = None, +) -> int: r''' Convert the given input to an integer or return default @@ -29,6 +35,10 @@ def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): 0 >>> to_int('1') 1 + >>> to_int('') + 0 + >>> to_int() + 0 >>> to_int('abc123') 0 >>> to_int('123abc') @@ -68,7 +78,6 @@ def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): ... TypeError: unknown argument for regexp parameter: 123 ''' - if regexp is True: regexp = re.compile(r'(\d+)') elif isinstance(regexp, str): @@ -83,13 +92,21 @@ def to_int(input_, default=0, exception=(ValueError, TypeError), regexp=None): match = regexp.search(input_) if match: input_ = match.groups()[-1] - return int(input_) - except exception: + + if input_ is None: + return default + else: + return int(input_) + except exception: # type: ignore return default -def to_float(input_, default=0, exception=(ValueError, TypeError), - regexp=None): +def to_float( + input_: str, + default: int = 0, + exception: types.ExceptionsType = (ValueError, TypeError), + regexp: types.O[re.Pattern] = None, +) -> types.Number: r''' Convert the given `input_` to an integer or return default @@ -161,7 +178,11 @@ def to_float(input_, default=0, exception=(ValueError, TypeError), return default -def to_unicode(input_, encoding='utf-8', errors='replace'): +def to_unicode( + input_: types.StringTypes, + encoding: str = 'utf-8', + errors: str = 'replace', +) -> str: '''Convert objects to unicode, if needed decodes string with the given encoding and errors settings. @@ -186,7 +207,11 @@ def to_unicode(input_, encoding='utf-8', errors='replace'): return input_ -def to_str(input_, encoding='utf-8', errors='replace'): +def to_str( + input_: types.StringTypes, + encoding: str = 'utf-8', + errors: str = 'replace', +) -> bytes: '''Convert objects to string, encodes to the given encoding :rtype: str @@ -213,7 +238,9 @@ def to_str(input_, encoding='utf-8', errors='replace'): return input_ -def scale_1024(x, n_prefixes): +def scale_1024( + x: types.Number, n_prefixes: int, +) -> types.Tuple[types.Number, types.Number]: '''Scale a number down to a suitable size, based on powers of 1024. Returns the scaled number and the power of 1024 used. @@ -239,7 +266,11 @@ def scale_1024(x, n_prefixes): return scaled, power -def remap(value, old_min, old_max, new_min, new_max): +def remap( + value: types.DecimalNumber, + old_min: types.DecimalNumber, old_max: types.DecimalNumber, + new_min: types.DecimalNumber, new_max: types.DecimalNumber, +) -> types.DecimalNumber: ''' remap a value from one range into another. @@ -312,7 +343,7 @@ def remap(value, old_min, old_max, new_min, new_max): :rtype: int, float, decimal.Decimal ''' - + type_: types.Type[types.DecimalNumber] if ( isinstance(value, decimal.Decimal) or isinstance(old_min, decimal.Decimal) or @@ -339,8 +370,8 @@ def remap(value, old_min, old_max, new_min, new_max): new_max = type_(new_max) new_min = type_(new_min) - old_range = old_max - old_min - new_range = new_max - new_min + old_range = old_max - old_min # type: ignore + new_range = new_max - new_min # type: ignore if old_range == 0: raise ValueError('Input range ({}-{}) is empty'.format( @@ -350,13 +381,13 @@ def remap(value, old_min, old_max, new_min, new_max): raise ValueError('Output range ({}-{}) is empty'.format( new_min, new_max)) - new_value = (value - old_min) * new_range + new_value = (value - old_min) * new_range # type: ignore if type_ == int: - new_value //= old_range + new_value //= old_range # type: ignore else: - new_value /= old_range + new_value /= old_range # type: ignore - new_value += new_min + new_value += new_min # type: ignore return new_value diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 06dccac..4f1cd96 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -1,7 +1,8 @@ import datetime +from typing import Union -def camel_to_underscore(name): +def camel_to_underscore(name: str) -> str: '''Convert camel case style naming to underscore style naming If there are existing underscores they will be collapsed with the @@ -39,7 +40,8 @@ def camel_to_underscore(name): return ''.join(output) -def timesince(dt, default='just now'): +def timesince(dt: Union[datetime.datetime, datetime.timedelta], + default: str = 'just now') -> str: ''' Returns string representing 'time since' e.g. 3 days ago, 5 hours ago etc. @@ -110,4 +112,3 @@ def timesince(dt, default='just now'): return '%s ago' % ' and '.join(output[:2]) return default - diff --git a/python_utils/import_.py b/python_utils/import_.py index 61091d8..c50da02 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -1,11 +1,18 @@ +from . import types + class DummyException(Exception): pass def import_global( - name, modules=None, exceptions=DummyException, locals_=None, - globals_=None, level=-1): + name: str, + modules: types.Optional[types.List[str]] = None, + exceptions: types.ExceptionsType = DummyException, + locals_: types.OptionalScope = None, + globals_: types.OptionalScope = None, + level: int = -1, +) -> types.Any: '''Import the requested items into the global scope WARNING! this method _will_ overwrite your global scope @@ -22,6 +29,8 @@ def import_global( relative imports ''' frame = None + name_parts: types.List[str] = name.split('.') + modules_set: types.Set[str] = set() try: # If locals_ or globals_ are not given, autodetect them by inspecting # the current stack @@ -36,44 +45,42 @@ def import_global( globals_ = frame.f_globals try: - name = name.split('.') - # Relative imports are supported (from .spam import eggs) - if not name[0]: - name = name[1:] + if not name_parts[0]: + name_parts = name_parts[1:] level = 1 # raise IOError((name, level)) module = __import__( - name=name[0] or '.', + name=name_parts[0] or '.', globals=globals_, locals=locals_, - fromlist=name[1:], + fromlist=name_parts[1:], level=max(level, 0), ) # Make sure we get the right part of a dotted import (i.e. # spam.eggs should return eggs, not spam) try: - for attr in name[1:]: + for attr in name_parts[1:]: module = getattr(module, attr) except AttributeError: - raise ImportError('No module named ' + '.'.join(name)) + raise ImportError('No module named ' + '.'.join(name_parts)) # If no list of modules is given, autodetect from either __all__ # or a dir() of the module if not modules: - modules = getattr(module, '__all__', dir(module)) + modules_set = set(getattr(module, '__all__', dir(module))) else: - modules = set(modules).intersection(dir(module)) + modules_set = set(modules).intersection(dir(module)) # Add all items in modules to the global scope - for k in set(dir(module)).intersection(modules): + for k in set(dir(module)).intersection(modules_set): if k and k[0] != '_': globals_[k] = getattr(module, k) except exceptions as e: return e finally: # Clean up, just to be sure - del name, modules, exceptions, locals_, globals_, frame - + del name, name_parts, modules, modules_set, exceptions, locals_, \ + globals_, frame diff --git a/python_utils/logger.py b/python_utils/logger.py index a5b6526..e44a8d8 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -1,8 +1,10 @@ -import logging import functools +import logging __all__ = ['Logged'] +import typing + class Logged(object): '''Class which automatically adds a named logger to your class when @@ -21,42 +23,44 @@ class Logged(object): >>> my_class.exception('exception') >>> my_class.log(0, 'log') ''' + + logger: logging.Logger + def __new__(cls, *args, **kwargs): cls.logger = logging.getLogger( cls.__get_name(cls.__module__, cls.__name__)) return super(Logged, cls).__new__(cls) @classmethod - def __get_name(cls, *name_parts): + def __get_name(cls, *name_parts: str) -> str: return '.'.join(n.strip() for n in name_parts if n.strip()) @classmethod @functools.wraps(logging.debug) - def debug(cls, msg, *args, **kwargs): + def debug(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): cls.logger.debug(msg, *args, **kwargs) @classmethod @functools.wraps(logging.info) - def info(cls, msg, *args, **kwargs): + def info(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): cls.logger.info(msg, *args, **kwargs) @classmethod @functools.wraps(logging.warning) - def warning(cls, msg, *args, **kwargs): + def warning(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): cls.logger.warning(msg, *args, **kwargs) @classmethod @functools.wraps(logging.error) - def error(cls, msg, *args, **kwargs): + def error(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): cls.logger.error(msg, *args, **kwargs) @classmethod @functools.wraps(logging.exception) - def exception(cls, msg, *args, **kwargs): + def exception(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): cls.logger.exception(msg, *args, **kwargs) @classmethod @functools.wraps(logging.log) - def log(cls, lvl, msg, *args, **kwargs): + def log(cls, lvl: int, msg: str, *args: typing.Any, **kwargs: typing.Any): cls.logger.log(lvl, msg, *args, **kwargs) - diff --git a/python_utils/terminal.py b/python_utils/terminal.py index d51e8ed..4cf0b4b 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -1,7 +1,10 @@ import os +import typing +from . import converters -def get_terminal_size(): # pragma: no cover + +def get_terminal_size() -> typing.Tuple[int, int]: # pragma: no cover '''Get the current size of your terminal Multiple returns are not always a good idea, but in this case it greatly @@ -11,12 +14,14 @@ def get_terminal_size(): # pragma: no cover Returns: width, height: Two integers containing width and height ''' + w: typing.Optional[int] + h: typing.Optional[int] try: # Default to 79 characters for IPython notebooks - from IPython import get_ipython + from IPython import get_ipython # type: ignore ipython = get_ipython() - from ipykernel import zmqshell + from ipykernel import zmqshell # type: ignore if isinstance(ipython, zmqshell.ZMQInteractiveShell): return 79, 24 except Exception: # pragma: no cover @@ -35,15 +40,15 @@ def get_terminal_size(): # pragma: no cover pass try: - w = int(os.environ.get('COLUMNS')) - h = int(os.environ.get('LINES')) + w = converters.to_int(os.environ.get('COLUMNS')) + h = converters.to_int(os.environ.get('LINES')) if w and h: return w, h except Exception: # pragma: no cover pass try: - import blessings + import blessings # type: ignore terminal = blessings.Terminal() w = terminal.width h = terminal.height diff --git a/python_utils/time.py b/python_utils/time.py index 4444ffd..65ecf9d 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -1,10 +1,10 @@ -from __future__ import absolute_import -import time -import typing import asyncio import datetime import itertools -import warnings +import typing + +from . import aio +import time delta_type = typing.Union[datetime.timedelta, int, float] timestamp_type = typing.Union[ @@ -17,7 +17,6 @@ None, ] - # There might be a better way to get the epoch with tzinfo, please create # a pull request if you know a better way that functions for Python 2 and 3 epoch = datetime.datetime(year=1970, month=1, day=1) @@ -114,14 +113,17 @@ def format_time(timestamp: timestamp_type, def timeout_generator( - timeout: delta_type, - interval: delta_type = datetime.timedelta(seconds=1), - iterable: typing.Iterable = itertools.count, - interval_multiplier: float = 1.0, + timeout: delta_type, + interval: delta_type = datetime.timedelta(seconds=1), + iterable: typing.Union[typing.Iterable, typing.Callable] = + itertools.count, + interval_multiplier: float = 1.0, + maximum_interval: typing.Optional[delta_type] = None, ): ''' Generator that walks through the given iterable (a counter by default) - until the timeout is reached with a configurable interval between items + until the float_timeout is reached with a configurable float_interval + between items >>> for i in timeout_generator(0.1, 0.06): ... print(i) @@ -146,69 +148,106 @@ def timeout_generator( ... print(i) 0 1 + 2 ''' + float_timeout: float = _to_seconds(timeout) + float_interval: float = _to_seconds(interval) + float_maximum_interval: typing.Optional[float] = _to_seconds_or_none( + maximum_interval) - if isinstance(interval, datetime.timedelta): - interval = timedelta_to_seconds(interval) - - if isinstance(timeout, datetime.timedelta): - timeout = timedelta_to_seconds(timeout) - + iterable_: typing.Iterable if callable(iterable): - iterable = iterable() + iterable_ = iterable() + else: + iterable_ = iterable - end = timeout + time.perf_counter() - for item in iterable: + end = float_timeout + time.perf_counter() + for item in iterable_: yield item if time.perf_counter() >= end: break + time.sleep(float_interval) + interval *= interval_multiplier - time.sleep(interval) + if float_maximum_interval: + float_interval = min(float_interval, float_maximum_interval) async def aio_timeout_generator( timeout: delta_type, interval: delta_type = datetime.timedelta(seconds=1), - iterable: typing.Iterable = itertools.count, + iterable: typing.Union[typing.AsyncIterable, typing.Callable] = + aio.acount, interval_multiplier: float = 1.0, - maximum_interval: delta_type = None, + maximum_interval: typing.Optional[delta_type] = None, ): ''' - Aync generator that walks through the given iterable (a counter by default) - until the timeout is reached with a configurable interval between items + Aync generator that walks through the given iterable (a counter by + default) until the float_timeout is reached with a configurable + float_interval between items - The interval_exponent automatically increases the timeout with each run. - Note that if the interval is less than 1, 1/interval_exponent will be used - so the interval is always growing. To double the interval with each run, - specify 2. + The interval_exponent automatically increases the float_timeout with each + run. Note that if the float_interval is less than 1, 1/interval_exponent + will be used so the float_interval is always growing. To double the + float_interval with each run, specify 2. Doctests and asyncio are not friends, so no examples. But this function is effectively the same as the timeout_generor but it uses `async for` instead. ''' + float_timeout: float = _to_seconds(timeout) + float_interval: float = _to_seconds(interval) + float_maximum_interval: typing.Optional[float] = _to_seconds_or_none( + maximum_interval) - if isinstance(interval, datetime.timedelta): - interval: int = timedelta_to_seconds(interval) - - if isinstance(maximum_interval, datetime.timedelta): - maximum_interval: int = timedelta_to_seconds(maximum_interval) - - if isinstance(timeout, datetime.timedelta): - timeout = timedelta_to_seconds(timeout) - + iterable_: typing.AsyncIterable if callable(iterable): - iterable = iterable() + iterable_ = iterable() + else: + iterable_ = iterable - end = timeout + time.perf_counter() - for item in iterable: + end = float_timeout + time.perf_counter() + async for item in iterable_: yield item if time.perf_counter() >= end: break - await asyncio.sleep(interval) - interval *= interval_multiplier - if maximum_interval: - interval = min(interval, maximum_interval) + await asyncio.sleep(float_interval) + + float_interval *= interval_multiplier + if float_maximum_interval: + float_interval = min(float_interval, float_maximum_interval) + + +def _to_seconds(interval: delta_type) -> float: + ''' + Convert a timedelta to seconds + + >>> _to_seconds(datetime.timedelta(seconds=1)) + 1 + >>> _to_seconds(datetime.timedelta(seconds=1, microseconds=1)) + 1.000001 + >>> _to_seconds(1) + 1 + >>> _to_seconds('whatever') # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: Unknown type ... + ''' + if isinstance(interval, datetime.timedelta): + return timedelta_to_seconds(interval) + elif isinstance(interval, (int, float)): + return interval + else: + raise TypeError('Unknown type %s: %r' % (type(interval), interval)) + + +def _to_seconds_or_none(interval: typing.Optional[delta_type]) -> \ + typing.Optional[float]: + if interval is None: + return None + else: + return _to_seconds(interval) diff --git a/setup.cfg b/setup.cfg index fb5b3dd..cc55116 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,3 +29,7 @@ universal = 1 [upload] sign = 1 +[flake8] +per-file-ignores = + python_utils/types.py: F403,F405 +exclude = From fd568ce5c7febeae1488dbe190643065d23c95f8 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 13:50:35 +0100 Subject: [PATCH 005/132] Added listify decorator for generators --- python_utils/decorators.py | 64 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index fc3e83a..be598f5 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -1,3 +1,5 @@ +import functools +from . import types def set_attributes(**kwargs): @@ -23,8 +25,70 @@ def set_attributes(**kwargs): >>> upper_case_name.short_description = 'Name' ''' + def _set_attributes(function): for key, value in kwargs.items(): setattr(function, key, value) return function + return _set_attributes + + +def listify(collection: types.Callable = list, allow_empty: bool = True): + ''' + Convert any generator to a list or other type of collection. + + >>> @listify() + ... def generator(): + ... yield 1 + ... yield 2 + ... yield 3 + + >>> generator() + [1, 2, 3] + + >>> @listify() + ... def empty_generator(): + ... pass + + >>> empty_generator() + [] + + >>> @listify(allow_empty=False) + ... def empty_generator_not_allowed(): + ... pass + + >>> empty_generator_not_allowed() + Traceback (most recent call last): + ... + TypeError: 'NoneType' object is not iterable + + >>> @listify(collection=set) + ... def set_generator(): + ... yield 1 + ... yield 1 + ... yield 2 + + >>> set_generator() + {1, 2} + + >>> @listify(collection=dict) + ... def dict_generator(): + ... yield 'a', 1 + ... yield 'b', 2 + + >>> dict_generator() + {'a': 1, 'b': 2} + ''' + + def _listify(function): + @functools.wraps(function) + def __listify(*args, **kwargs): + result = function(*args, **kwargs) + if result is None and allow_empty: + return [] + return collection(result) + + return __listify + + return _listify From d79b431d8b368232eaa256089a5ffd236eb90471 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 13:51:14 +0100 Subject: [PATCH 006/132] Added typing shortcuts and definitions --- python_utils/types.py | 124 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 124 insertions(+) create mode 100644 python_utils/types.py diff --git a/python_utils/types.py b/python_utils/types.py new file mode 100644 index 0000000..e2f64e9 --- /dev/null +++ b/python_utils/types.py @@ -0,0 +1,124 @@ +import decimal +from typing import * # pragma: no cover + +# Quickhand for optional because it gets so much use. If only Python had +# support for an optional type shorthand such as `SomeType?` instead of +# `Optional[SomeType]`. +from typing import Optional as O +# Since the Union operator is only supported for Python 3.10, we'll create a +# shorthand for it. +from typing import Union as U + +Scope = Dict[str, Any] +OptionalScope = O[Scope] +Number = U[int, float] +DecimalNumber = U[Number, decimal.Decimal] +ExceptionType = Type[Exception] +ExceptionsType = U[Tuple[ExceptionType, ...], ExceptionType] +StringTypes = U[str, bytes] + +assert Callable + +__all__ = [ + 'OptionalScope', + 'Number', + 'DecimalNumber', + + # The types from the typing module. + + # Super-special typing primitives. + 'Annotated', + 'Any', + 'Callable', + 'ClassVar', + 'Concatenate', + 'Final', + 'ForwardRef', + 'Generic', + 'Literal', + 'Optional', + 'ParamSpec', + 'Protocol', + 'Tuple', + 'Type', + 'TypeVar', + 'Union', + + # ABCs (from collections.abc). + 'AbstractSet', # collections.abc.Set. + 'ByteString', + 'Container', + 'ContextManager', + 'Hashable', + 'ItemsView', + 'Iterable', + 'Iterator', + 'KeysView', + 'Mapping', + 'MappingView', + 'MutableMapping', + 'MutableSequence', + 'MutableSet', + 'Sequence', + 'Sized', + 'ValuesView', + 'Awaitable', + 'AsyncIterator', + 'AsyncIterable', + 'Coroutine', + 'Collection', + 'AsyncGenerator', + 'AsyncContextManager', + + # Structural checks, a.k.a. protocols. + 'Reversible', + 'SupportsAbs', + 'SupportsBytes', + 'SupportsComplex', + 'SupportsFloat', + 'SupportsIndex', + 'SupportsInt', + 'SupportsRound', + + # Concrete collection types. + 'ChainMap', + 'Counter', + 'Deque', + 'Dict', + 'DefaultDict', + 'List', + 'OrderedDict', + 'Set', + 'FrozenSet', + 'NamedTuple', # Not really a type. + 'TypedDict', # Not really a type. + 'Generator', + + # Other concrete types. + 'BinaryIO', + 'IO', + 'Match', + 'Pattern', + 'TextIO', + + # One-off things. + 'AnyStr', + 'cast', + 'final', + 'get_args', + 'get_origin', + 'get_type_hints', + 'is_typeddict', + 'NewType', + 'no_type_check', + 'no_type_check_decorator', + 'NoReturn', + 'overload', + 'ParamSpecArgs', + 'ParamSpecKwargs', + 'runtime_checkable', + 'Text', + 'TYPE_CHECKING', + 'TypeAlias', + 'TypeGuard', +] From 6ab234dc9de2499e60666f7740deb5010483e8c3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 13:51:42 +0100 Subject: [PATCH 007/132] Added asyncio version of itertools.count for async for ... in count --- python_utils/aio.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 python_utils/aio.py diff --git a/python_utils/aio.py b/python_utils/aio.py new file mode 100644 index 0000000..28a3f01 --- /dev/null +++ b/python_utils/aio.py @@ -0,0 +1,9 @@ +import asyncio +import itertools + + +async def acount(start=0, step=1, delay=0): + '''Asyncio version of itertools.count()''' + for item in itertools.count(start, step): # pragma: no branch + yield item + await asyncio.sleep(delay) From f9e43f5f0b6c21615375b2f0bdb531894088f1f5 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 13:54:42 +0100 Subject: [PATCH 008/132] removed duplicate python requires --- .github/workflows/main.yml | 2 +- setup.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 437e61a..53feced 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 2 strategy: matrix: - python-version: [2.7, 3.6, 3.7, 3.8, 3.9, '3.10'] + python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10'] steps: - uses: actions/checkout@v2 diff --git a/setup.py b/setup.py index 6910c18..0d8a674 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ '_python_utils_tests', '*.__pycache__']), long_description=long_description, tests_require=['pytest'], - python_requires='>3.5', extras_require={ 'docs': [ 'six', From 84de092369a5dbd38f09c11d3c6c4f4892e60ef7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 14:11:36 +0100 Subject: [PATCH 009/132] made tests more robust --- _python_utils_tests/test_time.py | 31 +++++++++++++------------------ tox.ini | 3 +-- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index a6e9f4a..1ff0f3f 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -1,5 +1,5 @@ -import datetime import itertools +from datetime import timedelta import pytest @@ -9,19 +9,15 @@ @pytest.mark.parametrize( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ - (0.01, 0.006, 0.5, 0.01, aio.acount, 2), - (0.01, 0.006, 0.5, 0.01, aio.acount, 2), - (0.01, 0.006, 0.5, 0.01, aio.acount(), 2), - (0.01, 0.006, 1.0, None, aio.acount, 2), - (datetime.timedelta(seconds=0.01), - datetime.timedelta(seconds=0.006), - 2.0, datetime.timedelta(seconds=0.01), aio.acount, - 2), + (0.01, 0.003, 0.5, 0.01, aio.acount, 3), + (0.02, 0.003, 0.5, 0.01, aio.acount(), 6), + (0.03, 0.003, 1.0, None, aio.acount, 9), + (timedelta(seconds=0.01), timedelta(seconds=0.006), + 2.0, timedelta(seconds=0.01), aio.acount, 2), ]) @pytest.mark.asyncio -async def test_aio_timeout_generator(iterable, timeout, interval, - interval_multiplier, - maximum_interval, result): +async def test_aio_timeout_generator(timeout, interval, interval_multiplier, + maximum_interval, iterable, result): i = None async for i in time.aio_timeout_generator( timeout, interval, iterable, @@ -37,14 +33,13 @@ async def test_aio_timeout_generator(iterable, timeout, interval, (0.01, 0.006, 0.5, 0.01, itertools.count, 2), (0.01, 0.006, 0.5, 0.01, itertools.count(), 2), (0.01, 0.006, 1.0, None, 'abc', 'c'), - (datetime.timedelta(seconds=0.01), - datetime.timedelta(seconds=0.006), - 2.0, datetime.timedelta(seconds=0.01), + (timedelta(seconds=0.01), + timedelta(seconds=0.006), + 2.0, timedelta(seconds=0.01), itertools.count, 2), ]) -def test_timeout_generator( - iterable, timeout, interval, interval_multiplier, - maximum_interval, result): +def test_timeout_generator(timeout, interval, interval_multiplier, + maximum_interval, iterable, result): i = None for i in time.timeout_generator( timeout=timeout, diff --git a/tox.ini b/tox.ini index 8601e25..11054be 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,9 @@ [tox] -envlist = py35, py36, py37, py38, py39, py310, pypy3, flake8, docs +envlist = py36, py37, py38, py39, py310, pypy3, flake8, docs skip_missing_interpreters = True [testenv] basepython = - py35: python3.5 py36: python3.6 py37: python3.7 py38: python3.8 From 04c8c249c983a0c0042bb3a72df436171a40a0d6 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 14:15:41 +0100 Subject: [PATCH 010/132] make tox more verbose --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 53feced..36146e3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,5 +23,6 @@ jobs: run: | python -m pip install --upgrade pip pip install tox tox-gh-actions + tox -l - name: Test with tox - run: tox + run: tox -vr From aacf706e1743826ff4580f0af049b833bf12cd42 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 14:20:29 +0100 Subject: [PATCH 011/132] tox does not appear to work with github actions, trying manual testing --- .github/workflows/main.yml | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 36146e3..d6ad154 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: tox +name: pytest on: push: @@ -11,18 +11,36 @@ jobs: timeout-minutes: 2 strategy: matrix: - python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10'] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools flake8 + pip install -e '.[tests]' + - name: flake8 + run: flake8 + - name: pytest + run: py.test + + docs: + runs-on: ubuntu-latest + timeout-minutes: 2 steps: - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} + - name: Set up Python uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} + python-version: '3.10' - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install tox tox-gh-actions - tox -l - - name: Test with tox - run: tox -vr + python -m pip install --upgrade pip setuptools + pip install -e '.[docs]' + - name: build docs + run: make html + working-directory: docs/ From 2d1cfc061d633104fe694e7f320ce269533f67d7 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 14:40:48 +0100 Subject: [PATCH 012/132] added compatibility with python 3.6 and 3.7 --- python_utils/converters.py | 9 ++------- python_utils/types.py | 4 +++- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/python_utils/converters.py b/python_utils/converters.py index fb6800f..3a8ca0c 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -1,8 +1,3 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - import decimal import math import re @@ -15,7 +10,7 @@ def to_int( input_: typing.Optional[str] = None, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.O[re.Pattern] = None, + regexp: types.O[types.Pattern] = None, ) -> int: r''' Convert the given input to an integer or return default @@ -105,7 +100,7 @@ def to_float( input_: str, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.O[re.Pattern] = None, + regexp: types.O[types.Pattern] = None, ) -> types.Number: r''' Convert the given `input_` to an integer or return default diff --git a/python_utils/types.py b/python_utils/types.py index e2f64e9..906dfc3 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,5 +1,7 @@ import decimal from typing import * # pragma: no cover +# import * does not import Pattern +from typing import Pattern # Quickhand for optional because it gets so much use. If only Python had # support for an optional type shorthand such as `SomeType?` instead of @@ -17,7 +19,7 @@ ExceptionsType = U[Tuple[ExceptionType, ...], ExceptionType] StringTypes = U[str, bytes] -assert Callable +assert Pattern __all__ = [ 'OptionalScope', From e09500c8ae683cb3e503105af99648d9646cfd7f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 14:43:52 +0100 Subject: [PATCH 013/132] making sure all modules are automatically imported --- python_utils/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python_utils/__init__.py b/python_utils/__init__.py index e69de29..10947b8 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -0,0 +1,10 @@ +from . import aio +from . import compat +from . import converters +from . import decorators +from . import formatters +from . import import_ +from . import logger +from . import terminal +from . import time +from . import types From 3445388a9286f6e0b984db656f6d8502cd8d7bb1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 14:45:18 +0100 Subject: [PATCH 014/132] making sure all modules are automatically imported --- python_utils/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 10947b8..0ec2d64 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -8,3 +8,16 @@ from . import terminal from . import time from . import types + +__all__ = [ + 'aio', + 'compat', + 'converters', + 'decorators', + 'formatters', + 'import_', + 'logger', + 'terminal', + 'time', + 'types', +] From 198b523cba7156b6a7683658eb0f645ab8da7191 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 14:55:30 +0100 Subject: [PATCH 015/132] fixed failing tests on slower systems --- _python_utils_tests/test_time.py | 10 +++++----- python_utils/time.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 1ff0f3f..9018ab6 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -9,11 +9,11 @@ @pytest.mark.parametrize( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ - (0.01, 0.003, 0.5, 0.01, aio.acount, 3), - (0.02, 0.003, 0.5, 0.01, aio.acount(), 6), - (0.03, 0.003, 1.0, None, aio.acount, 9), - (timedelta(seconds=0.01), timedelta(seconds=0.006), - 2.0, timedelta(seconds=0.01), aio.acount, 2), + (0.1, 0.06, 0.5, 0.1, aio.acount, 2), + (0.2, 0.06, 0.5, 0.1, aio.acount(), 4), + (0.3, 0.06, 1.0, None, aio.acount, 5), + (timedelta(seconds=0.1), timedelta(seconds=0.06), + 2.0, timedelta(seconds=0.1), aio.acount, 2), ]) @pytest.mark.asyncio async def test_aio_timeout_generator(timeout, interval, interval_multiplier, diff --git a/python_utils/time.py b/python_utils/time.py index 65ecf9d..4f06248 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -209,7 +209,7 @@ async def aio_timeout_generator( iterable_ = iterable end = float_timeout + time.perf_counter() - async for item in iterable_: + async for item in iterable_: # pragma: no branch yield item if time.perf_counter() >= end: @@ -218,7 +218,7 @@ async def aio_timeout_generator( await asyncio.sleep(float_interval) float_interval *= interval_multiplier - if float_maximum_interval: + if float_maximum_interval: # pragma: no branch float_interval = min(float_interval, float_maximum_interval) From 51286e655c105fe83b296cafe48eb6790ea254b4 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 15:08:43 +0100 Subject: [PATCH 016/132] added shortcuts for simpler imports --- python_utils/__init__.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 0ec2d64..3e31921 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -9,6 +9,24 @@ from . import time from . import types +from .aio import acount +from .converters import remap +from .converters import scale_1024 +from .converters import to_float +from .converters import to_int +from .converters import to_str +from .converters import to_unicode +from .decorators import listify +from .decorators import set_attributes +from .formatters import camel_to_underscore +from .formatters import timesince +from .import_ import import_global +from .terminal import get_terminal_size +from .time import format_time +from .time import timedelta_to_seconds +from .time import timeout_generator +from .time import aio_timeout_generator + __all__ = [ 'aio', 'compat', @@ -20,4 +38,21 @@ 'terminal', 'time', 'types', + 'to_int', + 'to_float', + 'to_unicode', + 'to_str', + 'scale_1024', + 'remap', + 'set_attributes', + 'listify', + 'camel_to_underscore', + 'timesince', + 'import_global', + 'get_terminal_size', + 'timedelta_to_seconds', + 'format_time', + 'timeout_generator', + 'acount', + 'aio_timeout_generator', ] From c3182df657cd0698a0de3a614c51e2d7a482c367 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 15:21:51 +0100 Subject: [PATCH 017/132] made flake8 call more specific --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d6ad154..3896168 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,7 +24,7 @@ jobs: python -m pip install --upgrade pip setuptools flake8 pip install -e '.[tests]' - name: flake8 - run: flake8 + run: flake8 python_utils setup.py - name: pytest run: py.test From 5193d20bb918f2b4119a81d11b37e85b2f28c611 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 15:23:11 +0100 Subject: [PATCH 018/132] added debug info to github action --- .github/workflows/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3896168..967216d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,6 +23,10 @@ jobs: run: | python -m pip install --upgrade pip setuptools flake8 pip install -e '.[tests]' + - name: Get versions + run: | + python -V + pip freeze - name: flake8 run: flake8 python_utils setup.py - name: pytest From f48ab663ed84e955178776bcff28f166e508cd77 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 15:24:32 +0100 Subject: [PATCH 019/132] made flake run more verbose --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 967216d..3b7595f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,7 +28,7 @@ jobs: python -V pip freeze - name: flake8 - run: flake8 python_utils setup.py + run: flake8 -v python_utils setup.py - name: pytest run: py.test From c34b381be680f27d076867f9769521a09716efb3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 15:46:03 +0100 Subject: [PATCH 020/132] omitting type info for automatic versioning script --- python_utils/__about__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 4a55392..49639f4 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,8 +1,9 @@ __package_name__: str = 'python-utils' -__version__: str = '2.7.1' __author__: str = 'Rick van Hattem' __author_email__: str = 'Wolph@wol.ph' __description__: str = ( 'Python Utils is a module with some convenient utilities not included ' 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' +# Omit type info due to automatic versioning script +__version__ = '2.7.1' From 2d2d374881d293266f6c7c575c9ba388e688a028 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 4 Jan 2022 15:47:31 +0100 Subject: [PATCH 021/132] Incrementing version to v3.0.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 49639f4..3efe1b2 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -6,4 +6,4 @@ 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '2.7.1' +__version__ = '3.0.0' From cc0ed0e08d949cc16b13a8f1269a1232a96f53a2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 14 Jan 2022 17:04:22 +0100 Subject: [PATCH 022/132] added casted dict class for easy automatic casting --- python_utils/containers.py | 179 +++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 python_utils/containers.py diff --git a/python_utils/containers.py b/python_utils/containers.py new file mode 100644 index 0000000..e5d5475 --- /dev/null +++ b/python_utils/containers.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import abc +from typing import Any +from typing import Generator + +from . import types + +KT = types.TypeVar('KT') +VT = types.TypeVar('VT') +DT = types.Dict[KT, VT] +KT_cast = types.Optional[types.Callable[[Any], KT]] +VT_cast = types.Optional[types.Callable[[Any], VT]] + +# Using types.Union instead of | since Python 3.7 doesn't fully support it +DictUpdateArgs = types.Union[ + types.Mapping, + types.Iterable[types.Union[types.Tuple[Any, Any], types.Mapping]], +] + + +class CastedDictBase(types.Dict[KT, VT], abc.ABC): + _key_cast: KT_cast + _value_cast: VT_cast + + def __init__( + self, + key_cast: KT_cast = None, + value_cast: VT_cast = None, + *args, + **kwargs + ) -> None: + self._value_cast = value_cast + self._key_cast = key_cast + self.update(*args, **kwargs) + + def update( + self, + *args: DictUpdateArgs, + **kwargs + ) -> None: + if args: + kwargs.update(*args) + + if kwargs: + for key, value in kwargs.items(): + self[key] = value + + def __setitem__(self, key: Any, value: Any) -> None: + if self._key_cast is not None: + key = self._key_cast(key) + + return super().__setitem__(key, value) + + +class CastedDict(CastedDictBase): + ''' + Custom dictionary that casts keys and values to the specified typing. + + Note that you can specify the types for mypy and type hinting with: + CastedDict[int, int](int, int) + + >>> d = CastedDict(int, int) + >>> d[1] = 2 + >>> d['3'] = '4' + >>> d.update({'5': '6'}) + >>> d.update([('7', '8')]) + >>> d + {1: 2, 3: 4, 5: 6, 7: 8} + >>> list(d.keys()) + [1, 3, 5, 7] + >>> list(d) + [1, 3, 5, 7] + >>> list(d.values()) + [2, 4, 6, 8] + >>> list(d.items()) + [(1, 2), (3, 4), (5, 6), (7, 8)] + >>> d[3] + 4 + + # Casts are optional and can be disabled by passing None as the cast + >>> d = CastedDict() + >>> d[1] = 2 + >>> d['3'] = '4' + >>> d.update({'5': '6'}) + >>> d.update([('7', '8')]) + >>> d + {1: 2, '3': '4', '5': '6', '7': '8'} + ''' + + def __setitem__(self, key, value): + if self._value_cast is not None: + value = self._value_cast(value) + + super().__setitem__(key, value) + + +class LazyCastedDict(CastedDictBase): + ''' + Custom dictionary that casts keys and lazily casts values to the specified + typing. Note that the values are cast only when they are accessed and + are not cached between executions. + + Note that you can specify the types for mypy and type hinting with: + LazyCastedDict[int, int](int, int) + + >>> d = LazyCastedDict(int, int) + >>> d[1] = 2 + >>> d['3'] = '4' + >>> d.update({'5': '6'}) + >>> d.update([('7', '8')]) + >>> d + {1: 2, 3: '4', 5: '6', 7: '8'} + >>> list(d.keys()) + [1, 3, 5, 7] + >>> list(d) + [1, 3, 5, 7] + >>> list(d.values()) + [2, 4, 6, 8] + >>> list(d.items()) + [(1, 2), (3, 4), (5, 6), (7, 8)] + >>> d[3] + 4 + + # Casts are optional and can be disabled by passing None as the cast + >>> d = LazyCastedDict() + >>> d[1] = 2 + >>> d['3'] = '4' + >>> d.update({'5': '6'}) + >>> d.update([('7', '8')]) + >>> d + {1: 2, '3': '4', '5': '6', '7': '8'} + >>> list(d.keys()) + [1, '3', '5', '7'] + >>> list(d.values()) + [2, '4', '6', '8'] + + >>> list(d.items()) + [(1, 2), ('3', '4'), ('5', '6'), ('7', '8')] + >>> d['3'] + '4' + ''' + + def __setitem__(self, key, value): + if self._key_cast is not None: + key = self._key_cast(key) + + super().__setitem__(key, value) + + def __getitem__(self, key) -> VT: + if self._key_cast is not None: + key = self._key_cast(key) + + value = super().__getitem__(key) + + if self._value_cast is not None: + value = self._value_cast(value) + + return value + + def items(self) -> Generator[tuple[KT, VT], None, None]: # type: ignore + if self._value_cast is None: + yield from super().items() + else: + for key, value in super().items(): + yield key, self._value_cast(value) + + def values(self) -> Generator[VT, None, None]: # type: ignore + if self._value_cast is None: + yield from super().values() + else: + for value in super().values(): + yield self._value_cast(value) + + +if __name__ == '__main__': + import doctest + + doctest.testmod() From 75302f4a2f7f7a7a854f0c5931b81e76e0e068e6 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 14 Jan 2022 17:04:46 +0100 Subject: [PATCH 023/132] dropped python 3.6 support because it is deprecated and so we can somewhat use | as type union --- .github/workflows/main.yml | 2 +- setup.py | 1 - tox.ini | 3 +-- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3b7595f..e5f0cdf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 2 strategy: matrix: - python-version: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python-version: ['3.7', '3.8', '3.9', '3.10'] steps: - uses: actions/checkout@v2 diff --git a/setup.py b/setup.py index 0d8a674..26166c3 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,6 @@ tests_require=['pytest'], extras_require={ 'docs': [ - 'six', 'mock', 'sphinx', 'python-utils', diff --git a/tox.ini b/tox.ini index 11054be..47d0814 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,9 @@ [tox] -envlist = py36, py37, py38, py39, py310, pypy3, flake8, docs +envlist = py37, py38, py39, py310, pypy3, flake8, docs skip_missing_interpreters = True [testenv] basepython = - py36: python3.6 py37: python3.7 py38: python3.8 py39: python3.9 From 7ff8f9bdd3ae152495d8f1455227cd621a098878 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 14 Jan 2022 17:07:24 +0100 Subject: [PATCH 024/132] Incrementing version to v3.1.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 3efe1b2..469432e 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -6,4 +6,4 @@ 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.0.0' +__version__ = '3.1.0' From b34e299489987daa4728834252dab5bc99ebd9f5 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 12 May 2022 21:53:12 +0200 Subject: [PATCH 025/132] Fixed pytest issues. Fixes: #29 --- .coveragerc | 1 + .travis.yml | 79 -------------------------------------- pytest.ini | 2 + python_utils/containers.py | 10 +++-- python_utils/converters.py | 2 +- 5 files changed, 11 insertions(+), 83 deletions(-) delete mode 100644 .travis.yml diff --git a/.coveragerc b/.coveragerc index 524e15a..1e4e2d1 100644 --- a/.coveragerc +++ b/.coveragerc @@ -21,3 +21,4 @@ exclude_lines = raise NotImplementedError if 0: if __name__ == .__main__.: + if typing.TYPE_CHECKING: diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 88380c2..0000000 --- a/.travis.yml +++ /dev/null @@ -1,79 +0,0 @@ -dist: xenial -sudo: false -language: python - -env: - global: - - PIP_WHEEL_DIR=$HOME/.wheels - - PIP_FIND_LINKS=file://$PIP_WHEEL_DIR - -matrix: - include: - - python: '3.6' - env: TOXENV=docs - - python: '3.6' - env: TOXENV=flake8 - - python: '2.7' - env: TOXENV=py27 - - python: '3.5' - env: TOXENV=py35 - - python: '3.6' - env: TOXENV=py36 - - python: '3.7' - env: TOXENV=py37 - - python: '3.8' - env: TOXENV=py38 - - python: '3.9-dev' - env: TOXENV=py39 - - python: 'pypy' - env: TOXENV=pypy - # Added power support architecture - - arch: ppc64le - python: '3.6' - env: TOXENV=docs - - arch: ppc64le - python: '3.6' - env: TOXENV=flake8 - - arch: ppc64le - python: '2.7' - env: TOXENV=py27 - - arch: ppc64le - python: '3.5' - env: TOXENV=py35 - - arch: ppc64le - python: '3.6' - env: TOXENV=py36 - - arch: ppc64le - python: '3.7' - env: TOXENV=py37 - - arch: ppc64le - python: '3.8' - env: TOXENV=py38 - - arch: ppc64le - python: '3.9-dev' - env: TOXENV=py39 - -cache: - directories: - - $HOME/.wheels - -# command to install dependencies, e.g. pip install -r requirements.txt -install: - - mkdir -p $PIP_WHEEL_DIR - - pip wheel -r _python_utils_tests/requirements.txt - - pip install -e . - - pip install tox - -script: - - tox - -after_success: - - pip install codecov coveralls - - coveralls - - codecov - -notifications: - email: - on_success: never - on_failure: change - diff --git a/pytest.ini b/pytest.ini index b43425d..75036b6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -17,3 +17,5 @@ flake8-ignore = doctest_optionflags = ALLOW_UNICODE ALLOW_BYTES + +asyncio_mode = strict diff --git a/python_utils/containers.py b/python_utils/containers.py index e5d5475..00bb86b 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,11 +1,14 @@ from __future__ import annotations import abc -from typing import Any -from typing import Generator +import typing +from typing import Any, Generator from . import types +if typing.TYPE_CHECKING: + import _typeshed # noqa: F401 + KT = types.TypeVar('KT') VT = types.TypeVar('VT') DT = types.Dict[KT, VT] @@ -16,6 +19,7 @@ DictUpdateArgs = types.Union[ types.Mapping, types.Iterable[types.Union[types.Tuple[Any, Any], types.Mapping]], + '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] @@ -37,7 +41,7 @@ def __init__( def update( self, *args: DictUpdateArgs, - **kwargs + **kwargs: VT ) -> None: if args: kwargs.update(*args) diff --git a/python_utils/converters.py b/python_utils/converters.py index 3a8ca0c..2e3dc79 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -181,7 +181,7 @@ def to_unicode( '''Convert objects to unicode, if needed decodes string with the given encoding and errors settings. - :rtype: unicode + :rtype: str >>> to_unicode(b'a') 'a' From f8c17ec8e8f74b868bb90a1efd1490aa553f216b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 12 May 2022 21:54:19 +0200 Subject: [PATCH 026/132] Added `Logurud` class which exposes functions such as `self.info(...)` to classes with `loguru` as logger --- README.rst | 22 ++++++++++++++ _python_utils_tests/test_logger.py | 19 ++++++++++++ python_utils/__init__.py | 3 ++ python_utils/logger.py | 49 +++++++++++++++++++++++------- python_utils/loguru.py | 17 +++++++++++ setup.py | 4 +++ 6 files changed, 103 insertions(+), 11 deletions(-) create mode 100644 _python_utils_tests/test_logger.py create mode 100644 python_utils/loguru.py diff --git a/README.rst b/README.rst index 6018a27..3d8b63a 100644 --- a/README.rst +++ b/README.rst @@ -36,16 +36,22 @@ Installation: The package can be installed through `pip` (this is the recommended method): +.. code-block:: bash + pip install python-utils Or if `pip` is not available, `easy_install` should work as well: +.. code-block:: bash + easy_install python-utils Or download the latest release from Pypi (https://pypi.python.org/pypi/python-utils) or Github. Note that the releases on Pypi are signed with my GPG key (https://pgp.mit.edu/pks/lookup?op=vindex&search=0xE81444E9CE1F695D) and can be checked using GPG: +.. code-block:: bash + gpg --verify python-utils-.tar.gz.asc python-utils-.tar.gz Quickstart @@ -226,6 +232,22 @@ Or add a correclty named logger to your classes which can be easily accessed: import logging my_class.log(logging.ERROR, 'log') +Alternatively loguru is also supported. It is largely a drop-in replacement for the logging module which is a bit more convenient to configure: + +First install the extra loguru package: + +.. code-block:: bash + + pip install 'python-utils[loguru]' + +.. code-block:: python + + class MyClass(Logurud): + ... + +Now you can use the `Logurud` class to make functions such as `self.info()` +available. The benefit of this approach is that you can add extra context or +options to you specific loguru instance (i.e. `self.logger`): Convenient type aliases and some commonly used types: diff --git a/_python_utils_tests/test_logger.py b/_python_utils_tests/test_logger.py new file mode 100644 index 0000000..7c193b6 --- /dev/null +++ b/_python_utils_tests/test_logger.py @@ -0,0 +1,19 @@ +import pytest + +from python_utils.loguru import Logurud + + +loguru = pytest.importorskip('loguru') + + +def test_logurud(): + class MyClass(Logurud): + pass + + my_class = MyClass() + my_class.debug('debug') + my_class.info('info') + my_class.warning('warning') + my_class.error('error') + my_class.exception('exception') + my_class.log(0, 'log') diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 3e31921..be276df 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -26,6 +26,7 @@ from .time import timedelta_to_seconds from .time import timeout_generator from .time import aio_timeout_generator +from .logger import Logged, LoggerBase __all__ = [ 'aio', @@ -55,4 +56,6 @@ 'timeout_generator', 'acount', 'aio_timeout_generator', + 'Logged', + 'LoggerBase', ] diff --git a/python_utils/logger.py b/python_utils/logger.py index e44a8d8..d6913e5 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -1,3 +1,4 @@ +import abc import functools import logging @@ -6,15 +7,18 @@ import typing -class Logged(object): - '''Class which automatically adds a named logger to your class when - interiting +class LoggerBase(abc.ABC): + '''Class which automatically adds logging utilities to your class when + interiting. Expects `logger` to be a logging.Logger or compatible instance. Adds easy access to debug, info, warning, error, exception and log methods - >>> class MyClass(Logged): + >>> class MyClass(LoggerBase): + ... logger = logging.getLogger(__name__) + ... ... def __init__(self): ... Logged.__init__(self) + >>> my_class = MyClass() >>> my_class.debug('debug') >>> my_class.info('info') @@ -23,13 +27,9 @@ class Logged(object): >>> my_class.exception('exception') >>> my_class.log(0, 'log') ''' - - logger: logging.Logger - - def __new__(cls, *args, **kwargs): - cls.logger = logging.getLogger( - cls.__get_name(cls.__module__, cls.__name__)) - return super(Logged, cls).__new__(cls) + # Being a tad lazy here and not creating a Protocol. + # The actual classes define the correct type anyway + logger: typing.Any @classmethod def __get_name(cls, *name_parts: str) -> str: @@ -64,3 +64,30 @@ def exception(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): @functools.wraps(logging.log) def log(cls, lvl: int, msg: str, *args: typing.Any, **kwargs: typing.Any): cls.logger.log(lvl, msg, *args, **kwargs) + + +class Logged(LoggerBase): + '''Class which automatically adds a named logger to your class when + interiting + + Adds easy access to debug, info, warning, error, exception and log methods + + >>> class MyClass(Logged): + ... def __init__(self): + ... Logged.__init__(self) + >>> my_class = MyClass() + >>> my_class.debug('debug') + >>> my_class.info('info') + >>> my_class.warning('warning') + >>> my_class.error('error') + >>> my_class.exception('exception') + >>> my_class.log(0, 'log') + ''' + + logger: logging.Logger + + def __new__(cls, *args, **kwargs): + cls.logger = logging.getLogger( + cls._LoggerBase__get_name(cls.__module__, cls.__name__) + ) + return super(Logged, cls).__new__(cls) diff --git a/python_utils/loguru.py b/python_utils/loguru.py new file mode 100644 index 0000000..f670daa --- /dev/null +++ b/python_utils/loguru.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from . import logger + +import loguru + +__all__ = ['Logurud'] + + +class Logurud(logger.LoggerBase): + logger: loguru.Logger + + def __new__(cls, *args, **kwargs): + # Import at runtime to make loguru an optional dependency + import loguru + cls.logger: loguru.Loguru = loguru.logger + return super().__new__(cls) diff --git a/setup.py b/setup.py index 26166c3..5f168e6 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,9 @@ long_description=long_description, tests_require=['pytest'], extras_require={ + 'loguru': [ + 'loguru', + ], 'docs': [ 'mock', 'sphinx', @@ -43,6 +46,7 @@ 'pytest-asyncio', 'sphinx', 'types-setuptools', + 'loguru', ], }, classifiers=['License :: OSI Approved :: BSD License'], From 7824e028dfb61193c63f89fdef17c4be5c8af88e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 12 May 2022 21:54:38 +0200 Subject: [PATCH 027/132] Fixed Sphinx building issues and enabled nit-picky mode by default to fix #30 --- docs/conf.py | 5 ++++- tox.ini | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 3dacd3d..9d82d43 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -60,4 +60,7 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# html_static_path = ['_static'] + +intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} + diff --git a/tox.ini b/tox.ini index 47d0814..9a3904f 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ commands = mkdir -p docs/_static sphinx-apidoc -o docs/ python_utils rm -f docs/modules.rst - sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} + sphinx-build -n -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} deps = -r{toxinidir}/docs/requirements.txt [flake8] From 94b65593ad633811884047f7eb92a7e2ff8492ca Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 12 May 2022 21:55:04 +0200 Subject: [PATCH 028/132] added apply recursive method and improved camel to underscore --- python_utils/formatters.py | 45 ++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 4f1cd96..0e428f8 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -1,9 +1,10 @@ import datetime -from typing import Union + +from python_utils import types def camel_to_underscore(name: str) -> str: - '''Convert camel case style naming to underscore style naming + '''Convert camel case style naming to underscore/snake case style naming If there are existing underscores they will be collapsed with the to-be-added underscores. Multiple consecutive capital letters will not be @@ -40,8 +41,44 @@ def camel_to_underscore(name: str) -> str: return ''.join(output) -def timesince(dt: Union[datetime.datetime, datetime.timedelta], - default: str = 'just now') -> str: +def apply_recursive( + function: types.Callable[[str], str], + data: types.OptionalScope = None, + **kwargs +) -> types.OptionalScope: + ''' + Apply a function to all keys in a scope recursively + + >>> apply_recursive(camel_to_underscore, {'SpamEggsAndBacon': 'spam'}) + {'spam_eggs_and_bacon': 'spam'} + >>> apply_recursive(camel_to_underscore, {'SpamEggsAndBacon': { + ... 'SpamEggsAndBacon': 'spam', + ... }}) + {'spam_eggs_and_bacon': {'spam_eggs_and_bacon': 'spam'}} + + >>> a = {'a_b_c': 123, 'def': {'DeF': 456}} + >>> b = apply_recursive(camel_to_underscore, a) + >>> b + {'a_b_c': 123, 'def': {'de_f': 456}} + + >>> apply_recursive(camel_to_underscore, None) + ''' + if data is None: + return None + + elif isinstance(data, dict): + return { + function(key): apply_recursive(function, value, **kwargs) + for key, value in data.items() + } + else: + return data + + +def timesince( + dt: types.Union[datetime.datetime, datetime.timedelta], + default: str = 'just now' +) -> str: ''' Returns string representing 'time since' e.g. 3 days ago, 5 hours ago etc. From 80ce68eaba13eab2fc21aadbebccc583168d5927 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 12 May 2022 22:13:29 +0200 Subject: [PATCH 029/132] Removed travis from docs --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 3d8b63a..9924421 100644 --- a/README.rst +++ b/README.rst @@ -1,8 +1,8 @@ Useful Python Utils ============================================================================== -.. image:: https://travis-ci.org/WoLpH/python-utils.svg?branch=master - :target: https://travis-ci.org/WoLpH/python-utils +.. image:: https://github.com/WoLpH/python-utils/actions/workflows/main.yml/badge.svg?branch=master + :target: https://github.com/WoLpH/python-utils/actions/workflows/main.yml .. image:: https://coveralls.io/repos/WoLpH/python-utils/badge.svg?branch=master :target: https://coveralls.io/r/WoLpH/python-utils?branch=master From 6be881dbfa77f3ed52b78cde2af92b0a2f824628 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 12 May 2022 22:14:45 +0200 Subject: [PATCH 030/132] Incrementing version to v3.2.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 469432e..2faa90f 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -6,4 +6,4 @@ 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.1.0' +__version__ = '3.2.0' From b9d8475374b1addf3e1c59fd515a0c0a5a4b2a7b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 13 May 2022 01:20:39 +0200 Subject: [PATCH 031/132] Fixing backwards compatibility. This broke wolph/numpy-stl#195 --- python_utils/logger.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python_utils/logger.py b/python_utils/logger.py index d6913e5..dc17d4b 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -86,6 +86,10 @@ class Logged(LoggerBase): logger: logging.Logger + @classmethod + def __get_name(cls, *name_parts: str) -> str: + return cls._LoggerBase__get_name(*name_parts) + def __new__(cls, *args, **kwargs): cls.logger = logging.getLogger( cls._LoggerBase__get_name(cls.__module__, cls.__name__) From 25b2f2f67916781d533aaea601d1333d32cc92e4 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 13 May 2022 01:21:02 +0200 Subject: [PATCH 032/132] Incrementing version to v3.2.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 2faa90f..919f9d5 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -6,4 +6,4 @@ 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.2.0' +__version__ = '3.2.1' From 28075b6f7bf5ec3ca996ef1e2073ea9ffb246811 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 13 May 2022 01:43:15 +0200 Subject: [PATCH 033/132] Fixing backwards compatibility with test coverage. This broke wolph/numpy-stl#195 --- python_utils/logger.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python_utils/logger.py b/python_utils/logger.py index dc17d4b..ab1f914 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -75,6 +75,7 @@ class Logged(LoggerBase): >>> class MyClass(Logged): ... def __init__(self): ... Logged.__init__(self) + >>> my_class = MyClass() >>> my_class.debug('debug') >>> my_class.info('info') @@ -82,16 +83,19 @@ class Logged(LoggerBase): >>> my_class.error('error') >>> my_class.exception('exception') >>> my_class.log(0, 'log') + + >>> my_class._Logged__get_name('spam') + 'spam' ''' logger: logging.Logger @classmethod def __get_name(cls, *name_parts: str) -> str: - return cls._LoggerBase__get_name(*name_parts) + return LoggerBase._LoggerBase__get_name(*name_parts) # type: ignore def __new__(cls, *args, **kwargs): cls.logger = logging.getLogger( - cls._LoggerBase__get_name(cls.__module__, cls.__name__) + cls.__get_name(cls.__module__, cls.__name__) ) return super(Logged, cls).__new__(cls) From 3e00ab50b536f1a579a36794bdb111b5fa0a1888 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 13 May 2022 01:43:28 +0200 Subject: [PATCH 034/132] Incrementing version to v3.2.2 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 919f9d5..225634f 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -6,4 +6,4 @@ 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.2.1' +__version__ = '3.2.2' From 990ded8826d5f163a6940da92b8cc031bc1bf223 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 19 May 2022 15:21:29 +0200 Subject: [PATCH 035/132] Fixed loguru stack location --- python_utils/loguru.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python_utils/loguru.py b/python_utils/loguru.py index f670daa..0bf5040 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -11,7 +11,5 @@ class Logurud(logger.LoggerBase): logger: loguru.Logger def __new__(cls, *args, **kwargs): - # Import at runtime to make loguru an optional dependency - import loguru - cls.logger: loguru.Loguru = loguru.logger + cls.logger: loguru.Loguru = loguru.logger.opt(depth=1) return super().__new__(cls) From d91fd004e9cc7b2a7aa86a1d3f16c55e92c5ff4b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 19 May 2022 15:21:47 +0200 Subject: [PATCH 036/132] Incrementing version to v3.2.3 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 225634f..0ee04fe 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -6,4 +6,4 @@ 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.2.2' +__version__ = '3.2.3' From 25a334b03466848724653c4e728a1112f380394e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 19 May 2022 15:30:11 +0200 Subject: [PATCH 037/132] for some reason test coverage suddenly dropped here... --- python_utils/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/logger.py b/python_utils/logger.py index ab1f914..cbb751c 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -88,7 +88,7 @@ class Logged(LoggerBase): 'spam' ''' - logger: logging.Logger + logger: logging.Logger # pragma: no cover @classmethod def __get_name(cls, *name_parts: str) -> str: From a937f587a6a93067b63eb45e71ca8aeeef17d602 Mon Sep 17 00:00:00 2001 From: targhs <34231252+targhs@users.noreply.github.com> Date: Tue, 24 May 2022 00:37:34 +0530 Subject: [PATCH 038/132] Add sample decorator --- _python_utils_tests/test_decorators.py | 36 ++++++++++++++++++++++++++ python_utils/decorators.py | 29 +++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 _python_utils_tests/test_decorators.py diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py new file mode 100644 index 0000000..facfd43 --- /dev/null +++ b/_python_utils_tests/test_decorators.py @@ -0,0 +1,36 @@ +from unittest.mock import MagicMock + +import pytest + +from python_utils.decorators import sample + + +@pytest.fixture +def random(monkeypatch): + mock = MagicMock() + monkeypatch.setattr("python_utils.decorators.random.random", mock, raising=True) + return mock + + +def test_sample_called(random): + demo_function = MagicMock() + decorated = sample(0.5)(demo_function) + random.return_value = 0.4 + decorated() + random.return_value = 0.0 + decorated() + args = [1, 2] + kwargs = {"1": 1, "2": 2} + decorated(*args, **kwargs) + demo_function.assert_called_with(*args, **kwargs) + assert demo_function.call_count == 3 + + +def test_sample_not_called(random): + demo_function = MagicMock() + decorated = sample(0.5)(demo_function) + random.return_value = 0.5 + decorated() + random.return_value = 1.0 + decorated() + assert demo_function.call_count == 0 \ No newline at end of file diff --git a/python_utils/decorators.py b/python_utils/decorators.py index be598f5..588dec3 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -1,4 +1,6 @@ import functools +import logging +import random from . import types @@ -92,3 +94,30 @@ def __listify(*args, **kwargs): return __listify return _listify + + +def sample(sample_rate: float): + ''' + Limit calls to a function based on given sample rate. + Number of calls to the function will be roughly equal to + sample_rate percentage. + + Usage: + + >>> @sample(0.5) + ... def demo_function(*args, **kwargs): + ... return 1 + + Calls to *demo_function* will be limited to 50% approximatly. + + ''' + def _sample(function): + @functools.wraps(function) + def __sample(*args, **kwargs): + if random.random() < sample_rate: + return function(*args, **kwargs) + else: + logging.debug('Skipped execution of %r(%r, %r) due to sampling', function, args, kwargs) # noqa: E501 + + return __sample + return _sample From 97b87a75693db4b7eced82176d5cf25c0c2bde13 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 03:31:02 +0200 Subject: [PATCH 039/132] Added (asyncio) generators for automatic batching and timeout detection --- _python_utils_tests/test_generators.py | 36 +++++ _python_utils_tests/test_time.py | 121 +++++++++++++-- python_utils/__init__.py | 64 ++++---- python_utils/aio.py | 9 +- python_utils/exceptions.py | 26 ++++ python_utils/generators.py | 62 ++++++++ python_utils/time.py | 203 +++++++++++++++++-------- python_utils/types.py | 14 ++ 8 files changed, 430 insertions(+), 105 deletions(-) create mode 100644 _python_utils_tests/test_generators.py create mode 100644 python_utils/exceptions.py create mode 100644 python_utils/generators.py diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py new file mode 100644 index 0000000..c06996d --- /dev/null +++ b/_python_utils_tests/test_generators.py @@ -0,0 +1,36 @@ +import pytest + +import python_utils + + +@pytest.mark.asyncio +async def test_abatcher(): + async for batch in python_utils.abatcher(python_utils.acount(stop=9), 3): + assert len(batch) == 3 + + async for batch in python_utils.abatcher(python_utils.acount(stop=2), 3): + assert len(batch) == 2 + + +@pytest.mark.asyncio +async def test_abatcher_timed(): + batches = [] + async for batch in python_utils.abatcher( + python_utils.acount(stop=10, delay=0.04), + interval=0.1 + ): + batches.append(batch) + + assert len(batches) == 3 + assert sum(len(batch) for batch in batches) == 10 + + +def test_batcher(): + batch = [] + for batch in python_utils.batcher(range(9), 3): + assert len(batch) == 3 + + for batch in python_utils.batcher(range(4), 3): + pass + + assert len(batch) == 1 diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 9018ab6..c4ab346 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -1,27 +1,28 @@ +import asyncio import itertools from datetime import timedelta import pytest -from python_utils import aio -from python_utils import time +import python_utils @pytest.mark.parametrize( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ - (0.1, 0.06, 0.5, 0.1, aio.acount, 2), - (0.2, 0.06, 0.5, 0.1, aio.acount(), 4), - (0.3, 0.06, 1.0, None, aio.acount, 5), + (0.1, 0.06, 0.45, 0.1, python_utils.acount, 2), + (0.2, 0.06, 0.43, 0.1, python_utils.acount(), 4), + (0.3, 0.06, 1.0, None, python_utils.acount, 5), (timedelta(seconds=0.1), timedelta(seconds=0.06), - 2.0, timedelta(seconds=0.1), aio.acount, 2), + 2.0, timedelta(seconds=0.1), python_utils.acount, 2), ]) @pytest.mark.asyncio async def test_aio_timeout_generator(timeout, interval, interval_multiplier, maximum_interval, iterable, result): i = None - async for i in time.aio_timeout_generator( - timeout, interval, iterable, - maximum_interval=maximum_interval): + async for i in python_utils.aio_timeout_generator( + timeout, interval, iterable, + maximum_interval=maximum_interval + ): pass assert i == result @@ -41,13 +42,103 @@ async def test_aio_timeout_generator(timeout, interval, interval_multiplier, def test_timeout_generator(timeout, interval, interval_multiplier, maximum_interval, iterable, result): i = None - for i in time.timeout_generator( - timeout=timeout, - interval=interval, - interval_multiplier=interval_multiplier, - iterable=iterable, - maximum_interval=maximum_interval, + for i in python_utils.timeout_generator( + timeout=timeout, + interval=interval, + interval_multiplier=interval_multiplier, + iterable=iterable, + maximum_interval=maximum_interval, ): pass assert i == result + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector(): + async def generator(): + for i in range(10): + await asyncio.sleep(i / 100.0) + yield i + + detector = python_utils.aio_generator_timeout_detector + # Test regular timeout with reraise + with pytest.raises(asyncio.TimeoutError): + async for i in detector(generator(), 0.05): + pass + + # Test regular timeout with clean exit + async for i in detector(generator(), 0.05, on_timeout=None): + pass + + assert i == 4 + + # Test total timeout with reraise + with pytest.raises(asyncio.TimeoutError): + async for i in detector(generator(), total_timeout=0.1): + pass + + # Test total timeout with clean exit + async for i in detector(generator(), total_timeout=0.1, on_timeout=None): + pass + + assert i == 4 + + # Test stop iteration + async for i in detector(generator(), on_timeout=None): + pass + + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_decorator(): + # Test regular timeout with reraise + @python_utils.aio_generator_timeout_detector_decorator(timeout=0.05) + async def generator(): + for i in range(10): + await asyncio.sleep(i / 100.0) + yield i + + with pytest.raises(asyncio.TimeoutError): + async for i in generator(): + pass + + # Test regular timeout with clean exit + @python_utils.aio_generator_timeout_detector_decorator( + timeout=0.05, + on_timeout=None + ) + async def generator(): + for i in range(10): + await asyncio.sleep(i / 100.0) + yield i + + async for i in generator(): + pass + + assert i == 4 + + # Test total timeout with reraise + @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.1) + async def generator(): + for i in range(10): + await asyncio.sleep(i / 100.0) + yield i + + with pytest.raises(asyncio.TimeoutError): + async for i in generator(): + pass + + # Test total timeout with clean exit + @python_utils.aio_generator_timeout_detector_decorator( + total_timeout=0.1, + on_timeout=None + ) + async def generator(): + for i in range(10): + await asyncio.sleep(i / 100.0) + yield i + + async for i in generator(): + pass + + assert i == 4 diff --git a/python_utils/__init__.py b/python_utils/__init__.py index be276df..de72a02 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -1,35 +1,39 @@ -from . import aio -from . import compat -from . import converters -from . import decorators -from . import formatters -from . import import_ -from . import logger -from . import terminal -from . import time -from . import types - +from . import ( + aio, + compat, + converters, + decorators, + formatters, + generators, + import_, + logger, + terminal, + time, + types, +) from .aio import acount -from .converters import remap -from .converters import scale_1024 -from .converters import to_float -from .converters import to_int -from .converters import to_str -from .converters import to_unicode -from .decorators import listify -from .decorators import set_attributes -from .formatters import camel_to_underscore -from .formatters import timesince +from .converters import remap, scale_1024, to_float, to_int, to_str, to_unicode +from .decorators import listify, set_attributes +from .exceptions import raise_exception, reraise +from .formatters import camel_to_underscore, timesince +from .generators import abatcher, batcher from .import_ import import_global -from .terminal import get_terminal_size -from .time import format_time -from .time import timedelta_to_seconds -from .time import timeout_generator -from .time import aio_timeout_generator from .logger import Logged, LoggerBase +from .terminal import get_terminal_size +from .time import ( + aio_generator_timeout_detector, + aio_generator_timeout_detector_decorator, + aio_timeout_generator, + delta_to_seconds, + delta_to_seconds_or_none, + format_time, + timedelta_to_seconds, + timeout_generator, +) __all__ = [ 'aio', + 'generators', 'compat', 'converters', 'decorators', @@ -55,7 +59,15 @@ 'format_time', 'timeout_generator', 'acount', + 'abatcher', + 'batcher', 'aio_timeout_generator', + 'aio_generator_timeout_detector_decorator', + 'aio_generator_timeout_detector', + 'delta_to_seconds', + 'delta_to_seconds_or_none', + 'reraise', + 'raise_exception', 'Logged', 'LoggerBase', ] diff --git a/python_utils/aio.py b/python_utils/aio.py index 28a3f01..d099af6 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -1,9 +1,16 @@ +''' +Asyncio equivalents to regular Python functions. + +''' import asyncio import itertools -async def acount(start=0, step=1, delay=0): +async def acount(start=0, step=1, delay=0, stop=None): '''Asyncio version of itertools.count()''' for item in itertools.count(start, step): # pragma: no branch + if stop is not None and item >= stop: + break + yield item await asyncio.sleep(delay) diff --git a/python_utils/exceptions.py b/python_utils/exceptions.py new file mode 100644 index 0000000..14855c5 --- /dev/null +++ b/python_utils/exceptions.py @@ -0,0 +1,26 @@ +import typing + + +def raise_exception( + exception_class: typing.Type[Exception], + *args: typing.Any, + **kwargs: typing.Any, +) -> typing.Callable: + ''' + Returns a function that raises an exception of the given type with the + given arguments. + + >>> raise_exception(ValueError, 'spam')('eggs') + Traceback (most recent call last): + ... + ValueError: spam + ''' + + def raise_(*args_: typing.Any, **kwargs_: typing.Any) -> typing.Any: + raise exception_class(*args, **kwargs) + + return raise_ + + +def reraise(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: + raise diff --git a/python_utils/generators.py b/python_utils/generators.py new file mode 100644 index 0000000..f5527a8 --- /dev/null +++ b/python_utils/generators.py @@ -0,0 +1,62 @@ +import time + +import python_utils +from python_utils import types + + +async def abatcher( + generator: types.AsyncGenerator, + batch_size: types.Optional[int] = None, + interval: types.Optional[types.delta_type] = None, +): + ''' + Asyncio generator wrapper that returns items with a given batch size or + interval (whichever is reached first). + ''' + batch: list = [] + + assert batch_size or interval, 'Must specify either batch_size or interval' + + if interval: + interval_s = python_utils.delta_to_seconds(interval) + next_yield = time.perf_counter() + interval_s + else: + interval_s = 0 + next_yield = 0 + + while True: + try: + item = await generator.__anext__() + except StopAsyncIteration: + if batch: + yield batch + break + else: + batch.append(item) + + if batch_size is not None and len(batch) == batch_size: + yield batch + batch = [] + + if interval and batch and time.perf_counter() > next_yield: + yield batch + batch = [] + # Always set the next yield time to the current time. If the + # loop is running slow due to blocking functions we do not + # want to burst too much + next_yield = time.perf_counter() + interval_s + + +def batcher(iterable, batch_size): + ''' + Generator wrapper that returns items with a given batch size + ''' + batch = [] + for item in iterable: + batch.append(item) + if len(batch) == batch_size: + yield batch + batch = [] + + if batch: + yield batch diff --git a/python_utils/time.py b/python_utils/time.py index 4f06248..8745c9c 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -1,28 +1,18 @@ import asyncio import datetime +import functools import itertools -import typing - -from . import aio import time -delta_type = typing.Union[datetime.timedelta, int, float] -timestamp_type = typing.Union[ - datetime.timedelta, - datetime.date, - datetime.datetime, - str, - int, - float, - None, -] +import python_utils +from python_utils import aio, exceptions, types # There might be a better way to get the epoch with tzinfo, please create # a pull request if you know a better way that functions for Python 2 and 3 epoch = datetime.datetime(year=1970, month=1, day=1) -def timedelta_to_seconds(delta: datetime.timedelta): +def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: '''Convert a timedelta to seconds with the microseconds as fraction Note that this method has become largely obsolete with the @@ -48,8 +38,42 @@ def timedelta_to_seconds(delta: datetime.timedelta): return total -def format_time(timestamp: timestamp_type, - precision: datetime.timedelta = datetime.timedelta(seconds=1)): +def delta_to_seconds(interval: types.delta_type) -> float: + ''' + Convert a timedelta to seconds + + >>> delta_to_seconds(datetime.timedelta(seconds=1)) + 1 + >>> delta_to_seconds(datetime.timedelta(seconds=1, microseconds=1)) + 1.000001 + >>> delta_to_seconds(1) + 1 + >>> delta_to_seconds('whatever') # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + TypeError: Unknown type ... + ''' + if isinstance(interval, datetime.timedelta): + return timedelta_to_seconds(interval) + elif isinstance(interval, (int, float)): + return interval + else: + raise TypeError('Unknown type %s: %r' % (type(interval), interval)) + + +def delta_to_seconds_or_none( + interval: types.Optional[types.delta_type] +) -> types.Optional[float]: + if interval is None: + return None + else: + return delta_to_seconds(interval) + + +def format_time( + timestamp: types.timestamp_type, + precision: datetime.timedelta = datetime.timedelta(seconds=1) +) -> str: '''Formats timedelta/datetime/seconds >>> format_time('1') @@ -113,12 +137,12 @@ def format_time(timestamp: timestamp_type, def timeout_generator( - timeout: delta_type, - interval: delta_type = datetime.timedelta(seconds=1), - iterable: typing.Union[typing.Iterable, typing.Callable] = - itertools.count, - interval_multiplier: float = 1.0, - maximum_interval: typing.Optional[delta_type] = None, + timeout: types.delta_type, + interval: types.delta_type = datetime.timedelta(seconds=1), + iterable: types.Union[types.Iterable, types.Callable] = + itertools.count, + interval_multiplier: float = 1.0, + maximum_interval: types.Optional[types.delta_type] = None, ): ''' Generator that walks through the given iterable (a counter by default) @@ -150,12 +174,13 @@ def timeout_generator( 1 2 ''' - float_timeout: float = _to_seconds(timeout) - float_interval: float = _to_seconds(interval) - float_maximum_interval: typing.Optional[float] = _to_seconds_or_none( - maximum_interval) + float_timeout: float = delta_to_seconds(timeout) + float_interval: float = delta_to_seconds(interval) + float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( + maximum_interval + ) - iterable_: typing.Iterable + iterable_: types.Iterable if callable(iterable): iterable_ = iterable() else: @@ -176,12 +201,12 @@ def timeout_generator( async def aio_timeout_generator( - timeout: delta_type, - interval: delta_type = datetime.timedelta(seconds=1), - iterable: typing.Union[typing.AsyncIterable, typing.Callable] = - aio.acount, - interval_multiplier: float = 1.0, - maximum_interval: typing.Optional[delta_type] = None, + timeout: types.delta_type, + interval: types.delta_type = datetime.timedelta(seconds=1), + iterable: types.Union[ + types.AsyncIterable, types.Callable] = aio.acount, + interval_multiplier: float = 1.0, + maximum_interval: types.Optional[types.delta_type] = None, ): ''' Aync generator that walks through the given iterable (a counter by @@ -194,15 +219,16 @@ async def aio_timeout_generator( float_interval with each run, specify 2. Doctests and asyncio are not friends, so no examples. But this function is - effectively the same as the timeout_generor but it uses `async for` + effectively the same as the `timeout_generator` but it uses `async for` instead. ''' - float_timeout: float = _to_seconds(timeout) - float_interval: float = _to_seconds(interval) - float_maximum_interval: typing.Optional[float] = _to_seconds_or_none( - maximum_interval) + float_timeout: float = delta_to_seconds(timeout) + float_interval: float = delta_to_seconds(interval) + float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( + maximum_interval + ) - iterable_: typing.AsyncIterable + iterable_: types.AsyncIterable if callable(iterable): iterable_ = iterable() else: @@ -222,32 +248,83 @@ async def aio_timeout_generator( float_interval = min(float_interval, float_maximum_interval) -def _to_seconds(interval: delta_type) -> float: +async def aio_generator_timeout_detector( + generator: types.AsyncGenerator, + timeout: types.Optional[types.delta_type] = None, + total_timeout: types.Optional[types.delta_type] = None, + on_timeout: types.Optional[types.Callable] = exceptions.reraise, + **kwargs, +): ''' - Convert a timedelta to seconds - - >>> _to_seconds(datetime.timedelta(seconds=1)) - 1 - >>> _to_seconds(datetime.timedelta(seconds=1, microseconds=1)) - 1.000001 - >>> _to_seconds(1) - 1 - >>> _to_seconds('whatever') # doctest: +ELLIPSIS - Traceback (most recent call last): - ... - TypeError: Unknown type ... + This function is used to detect if an asyncio generator has not yielded + an element for a set amount of time. + + The `on_timeout` argument is called with the `generator`, `timeout`, + `total_timeout`, `exception` and the extra `**kwargs` to this function as + arguments. + If `on_timeout` is not specified, the exception is reraised. + If `on_timeout` is `None`, the exception is silently ignored and the + generator will finish as normal. ''' - if isinstance(interval, datetime.timedelta): - return timedelta_to_seconds(interval) - elif isinstance(interval, (int, float)): - return interval + if total_timeout is None: + total_timeout_end = None else: - raise TypeError('Unknown type %s: %r' % (type(interval), interval)) + total_timeout_end = time.perf_counter() + delta_to_seconds( + total_timeout + ) + timeout_s = python_utils.delta_to_seconds_or_none(timeout) -def _to_seconds_or_none(interval: typing.Optional[delta_type]) -> \ - typing.Optional[float]: - if interval is None: - return None - else: - return _to_seconds(interval) + while True: + try: + if total_timeout_end and time.perf_counter() >= total_timeout_end: + raise asyncio.TimeoutError('Total timeout reached') + + if timeout_s: + yield await asyncio.wait_for(generator.__anext__(), timeout_s) + else: + yield await generator.__anext__() + + except asyncio.TimeoutError as exception: + if on_timeout is not None: + await on_timeout( + generator, + timeout, + total_timeout, + exception, + **kwargs + ) + break + + except StopAsyncIteration: + break + + +def aio_generator_timeout_detector_decorator( + timeout: types.Optional[types.delta_type] = None, + total_timeout: types.Optional[types.delta_type] = None, + on_timeout: types.Optional[types.Callable] = exceptions.reraise, + **kwargs, +): + ''' + A decorator wrapper for aio_generator_timeout_detector. + ''' + + def _timeout_detector_decorator(generator: types.Callable): + ''' + The decorator itself. + ''' + + @functools.wraps(generator) + def wrapper(*args, **wrapper_kwargs): + return aio_generator_timeout_detector( + generator(*args, **wrapper_kwargs), + timeout, + total_timeout, + on_timeout, + **kwargs, + ) + + return wrapper + + return _timeout_detector_decorator diff --git a/python_utils/types.py b/python_utils/types.py index 906dfc3..282861f 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,3 +1,4 @@ +import datetime import decimal from typing import * # pragma: no cover # import * does not import Pattern @@ -19,12 +20,25 @@ ExceptionsType = U[Tuple[ExceptionType, ...], ExceptionType] StringTypes = U[str, bytes] +delta_type = U[datetime.timedelta, int, float] +timestamp_type = U[ + datetime.timedelta, + datetime.date, + datetime.datetime, + str, + int, + float, + None, +] + assert Pattern __all__ = [ 'OptionalScope', 'Number', 'DecimalNumber', + 'delta_type', + 'timestamp_type', # The types from the typing module. From f9b3014ceb697e14760aa0272adb3f89cbfc6535 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 03:46:22 +0200 Subject: [PATCH 040/132] fixed small mypy issues --- python_utils/converters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/converters.py b/python_utils/converters.py index 2e3dc79..ea47d94 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -83,7 +83,7 @@ def to_int( raise TypeError('unknown argument for regexp parameter: %r' % regexp) try: - if regexp: + if regexp and input_: match = regexp.search(input_) if match: input_ = match.groups()[-1] From dd47effc06c3aa0989d28da3987539f31babc83f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 04:00:32 +0200 Subject: [PATCH 041/132] made the tests more resilient --- _python_utils_tests/test_generators.py | 4 ++-- _python_utils_tests/test_time.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index c06996d..bfafb9e 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -16,8 +16,8 @@ async def test_abatcher(): async def test_abatcher_timed(): batches = [] async for batch in python_utils.abatcher( - python_utils.acount(stop=10, delay=0.04), - interval=0.1 + python_utils.acount(stop=10, delay=0.08), + interval=0.2 ): batches.append(batch) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index c4ab346..935002d 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -9,8 +9,8 @@ @pytest.mark.parametrize( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ - (0.1, 0.06, 0.45, 0.1, python_utils.acount, 2), - (0.2, 0.06, 0.43, 0.1, python_utils.acount(), 4), + (0.2, 0.1, 0.4, 0.2, python_utils.acount, 2), + (0.3, 0.1, 0.4, 0.2, python_utils.acount(), 3), (0.3, 0.06, 1.0, None, python_utils.acount, 5), (timedelta(seconds=0.1), timedelta(seconds=0.06), 2.0, timedelta(seconds=0.1), python_utils.acount, 2), From 102e603334997e7f4a0670617c036bdcc521bf34 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 04:07:42 +0200 Subject: [PATCH 042/132] Incrementing version to v3.3.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 0ee04fe..d1e6a30 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -6,4 +6,4 @@ 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.2.3' +__version__ = '3.3.0' From 29866a0d998edfb01568f45db3b9e4abbbcaee72 Mon Sep 17 00:00:00 2001 From: targhs <34231252+targhs@users.noreply.github.com> Date: Mon, 30 May 2022 02:04:01 +0530 Subject: [PATCH 043/132] Add CONTRIBUTING.md --- CONTRIBUTING.md | 61 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..b165d4c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,61 @@ +# Contributing to python-utils + +Bug reports, code and documentation contributions are welcome. You can help this +project also by using the development version and by reporting any bugs you might encounter + +## 1. Reporting bugs +It's important to provide following details when submitting a bug +- Python version +- python-utils version +- OS details + +If possible also provide a minimum reproducible working code. +## 2. Contributing Code and Docs + +Before working on a new feature or a bug, please browse [existing issues](https://github.com/WoLpH/python-utils/issues) +to see whether it has previously been discussed. + +If your change alters python-util's behaviour or interface, it's a good idea to +discuss it before you start working on it. + +If you are fixing an issue, the first step should be to create a test case that +reproduces the incorrect behaviour. That will also help you to build an +understanding of the issue at hand. + +Make sure to add relevant tests and update documentation in order to get +your PRs merged. We strictly adhere to 100% code coverage. + +### Development Environment + +#### Getting the code + +Go to and fork the project repository. + +```bash +# Clone your fork +$ git clone git@github.com:/python-utils.git + +# Enter the project directory +$ cd python-utils + +# Create a branch for your changes +$ git checkout -b my_awesome_branch +``` + +#### Testing +Before submitting any PR make sure your code passes all the tests. + +Create virtual environment and activate +``` +$ python3 -m venv venv +$ source venv/bin/activate +``` +Install test requirements +``` +$ cd python-utils +$ pip install -r _python_utils_tests/requirements.txt +``` +Run tests +``` +$ tox +``` \ No newline at end of file From 157c39b2235aff0a713c1e016c5a6c9166358b30 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 23:19:18 +0200 Subject: [PATCH 044/132] completely removed pytest-flake8 remnanets to fix #29 --- pytest.ini | 7 +------ setup.py | 1 - 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/pytest.ini b/pytest.ini index 75036b6..5d49701 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,15 +6,10 @@ python_files = addopts = --doctest-modules --cov python_utils - --cov-report html --cov-report term-missing --mypy -flake8-ignore = - *.py W391 - docs/*.py ALL - -doctest_optionflags = +doctest_optionflags = ALLOW_UNICODE ALLOW_BYTES diff --git a/setup.py b/setup.py index 5f168e6..e2c5962 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,6 @@ 'pytest', 'pytest-cov', 'pytest-mypy', - 'pytest-flake8', 'pytest-asyncio', 'sphinx', 'types-setuptools', From 3908ac833ba50be969e833622b214aaba7fcd15b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 23:19:49 +0200 Subject: [PATCH 045/132] improved generator timeout handling --- python_utils/generators.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python_utils/generators.py b/python_utils/generators.py index f5527a8..3bfc6db 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -1,3 +1,4 @@ +import asyncio import time import python_utils @@ -26,8 +27,13 @@ async def abatcher( while True: try: - item = await generator.__anext__() - except StopAsyncIteration: + if interval_s: + item = await asyncio.wait_for( + generator.__anext__(), interval_s + ) + else: + item = await generator.__anext__() + except (StopAsyncIteration, asyncio.TimeoutError): if batch: yield batch break From 62631f1f3f6cc7ff4df79d2340dd570029079305 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 23:20:01 +0200 Subject: [PATCH 046/132] Incrementing version to v3.3.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index d1e6a30..89c1486 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -6,4 +6,4 @@ 'with the standard Python install') __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.3.0' +__version__ = '3.3.1' From f36c1db394b8b2672f316b7f3e9809e95e8999e3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 23:32:20 +0200 Subject: [PATCH 047/132] Made contributing instructions slightly elaborate with regards to testing --- CONTRIBUTING.md | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b165d4c..b2e961e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,17 +45,37 @@ $ git checkout -b my_awesome_branch #### Testing Before submitting any PR make sure your code passes all the tests. -Create virtual environment and activate +To run the full test-suite, make sure you have `tox` installed and run the following command: + +```bash +$ tox +``` + +Or to speed it up (replace 8 with your number of cores), run: + +```bash +$ tox -p8 ``` + +During development I recommend using pytest directly and installing the package in development mode. + +Create virtual environment and activate +```bash $ python3 -m venv venv $ source venv/bin/activate ``` Install test requirements -``` +```bash $ cd python-utils -$ pip install -r _python_utils_tests/requirements.txt +$ pip install -e ".[tests]" ``` Run tests +```bash +$ py.test +``` + +Note that this won't run `flake8` yet, so once all the tests succeed you can run `flake8` to check for code style errors. + +```bash +$ flake8 ``` -$ tox -``` \ No newline at end of file From 02a2237becff401ead6e7a9515faba4e33662e39 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 29 May 2022 23:48:35 +0200 Subject: [PATCH 048/132] blacked the code --- _python_utils_tests/test_decorators.py | 8 +-- _python_utils_tests/test_generators.py | 3 +- _python_utils_tests/test_import.py | 17 +++--- _python_utils_tests/test_python_utils.py | 1 - _python_utils_tests/test_time.py | 53 +++++++++++------- python_utils/__about__.py | 3 +- python_utils/containers.py | 16 +++--- python_utils/converters.py | 69 +++++++++++++----------- python_utils/decorators.py | 9 +++- python_utils/formatters.py | 4 +- python_utils/import_.py | 25 ++++++--- python_utils/logger.py | 1 + python_utils/terminal.py | 29 +++++++--- python_utils/time.py | 16 ++---- python_utils/types.py | 9 +--- setup.py | 5 +- tox.ini | 9 +++- 17 files changed, 162 insertions(+), 115 deletions(-) diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py index facfd43..2ab6c1d 100644 --- a/_python_utils_tests/test_decorators.py +++ b/_python_utils_tests/test_decorators.py @@ -8,13 +8,15 @@ @pytest.fixture def random(monkeypatch): mock = MagicMock() - monkeypatch.setattr("python_utils.decorators.random.random", mock, raising=True) + monkeypatch.setattr( + "python_utils.decorators.random.random", mock, raising=True + ) return mock def test_sample_called(random): demo_function = MagicMock() - decorated = sample(0.5)(demo_function) + decorated = sample(0.5)(demo_function) random.return_value = 0.4 decorated() random.return_value = 0.0 @@ -33,4 +35,4 @@ def test_sample_not_called(random): decorated() random.return_value = 1.0 decorated() - assert demo_function.call_count == 0 \ No newline at end of file + assert demo_function.call_count == 0 diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index bfafb9e..25c2853 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -16,8 +16,7 @@ async def test_abatcher(): async def test_abatcher_timed(): batches = [] async for batch in python_utils.abatcher( - python_utils.acount(stop=10, delay=0.08), - interval=0.2 + python_utils.acount(stop=10, delay=0.08), interval=0.2 ): batches.append(batch) diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index d699e20..2834abc 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -17,7 +17,8 @@ def test_import_globals_without_inspection(): locals_ = {} globals_ = {'__name__': __name__} import_.import_global( - 'python_utils.formatters', locals_=locals_, globals_=globals_) + 'python_utils.formatters', locals_=locals_, globals_=globals_ + ) assert 'camel_to_underscore' in globals_ @@ -25,8 +26,11 @@ def test_import_globals_single_method(): locals_ = {} globals_ = {'__name__': __name__} import_.import_global( - 'python_utils.formatters', ['camel_to_underscore'], locals_=locals_, - globals_=globals_) + 'python_utils.formatters', + ['camel_to_underscore'], + locals_=locals_, + globals_=globals_, + ) assert 'camel_to_underscore' in globals_ @@ -37,12 +41,13 @@ def test_import_globals_with_inspection(): def test_import_globals_missing_module(): import_.import_global( - 'python_utils.spam', exceptions=ImportError, locals_=locals()) + 'python_utils.spam', exceptions=ImportError, locals_=locals() + ) assert 'camel_to_underscore' in globals() def test_import_locals_missing_module(): import_.import_global( - 'python_utils.spam', exceptions=ImportError, globals_=globals()) + 'python_utils.spam', exceptions=ImportError, globals_=globals() + ) assert 'camel_to_underscore' in globals() - diff --git a/_python_utils_tests/test_python_utils.py b/_python_utils_tests/test_python_utils.py index d6af258..0ced509 100644 --- a/_python_utils_tests/test_python_utils.py +++ b/_python_utils_tests/test_python_utils.py @@ -7,4 +7,3 @@ def test_definitions(): assert __about__.__author__ assert __about__.__author_email__ assert __about__.__description__ - diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 935002d..3da6284 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -8,20 +8,28 @@ @pytest.mark.parametrize( - 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ + 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', + [ (0.2, 0.1, 0.4, 0.2, python_utils.acount, 2), (0.3, 0.1, 0.4, 0.2, python_utils.acount(), 3), (0.3, 0.06, 1.0, None, python_utils.acount, 5), - (timedelta(seconds=0.1), timedelta(seconds=0.06), - 2.0, timedelta(seconds=0.1), python_utils.acount, 2), - ]) + ( + timedelta(seconds=0.1), + timedelta(seconds=0.06), + 2.0, + timedelta(seconds=0.1), + python_utils.acount, + 2, + ), + ], +) @pytest.mark.asyncio -async def test_aio_timeout_generator(timeout, interval, interval_multiplier, - maximum_interval, iterable, result): +async def test_aio_timeout_generator( + timeout, interval, interval_multiplier, maximum_interval, iterable, result +): i = None async for i in python_utils.aio_timeout_generator( - timeout, interval, iterable, - maximum_interval=maximum_interval + timeout, interval, iterable, maximum_interval=maximum_interval ): pass @@ -29,18 +37,25 @@ async def test_aio_timeout_generator(timeout, interval, interval_multiplier, @pytest.mark.parametrize( - 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ + 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', + [ (0.01, 0.006, 0.5, 0.01, 'abc', 'c'), (0.01, 0.006, 0.5, 0.01, itertools.count, 2), (0.01, 0.006, 0.5, 0.01, itertools.count(), 2), (0.01, 0.006, 1.0, None, 'abc', 'c'), - (timedelta(seconds=0.01), - timedelta(seconds=0.006), - 2.0, timedelta(seconds=0.01), - itertools.count, 2), - ]) -def test_timeout_generator(timeout, interval, interval_multiplier, - maximum_interval, iterable, result): + ( + timedelta(seconds=0.01), + timedelta(seconds=0.006), + 2.0, + timedelta(seconds=0.01), + itertools.count, + 2, + ), + ], +) +def test_timeout_generator( + timeout, interval, interval_multiplier, maximum_interval, iterable, result +): i = None for i in python_utils.timeout_generator( timeout=timeout, @@ -104,8 +119,7 @@ async def generator(): # Test regular timeout with clean exit @python_utils.aio_generator_timeout_detector_decorator( - timeout=0.05, - on_timeout=None + timeout=0.05, on_timeout=None ) async def generator(): for i in range(10): @@ -130,8 +144,7 @@ async def generator(): # Test total timeout with clean exit @python_utils.aio_generator_timeout_detector_decorator( - total_timeout=0.1, - on_timeout=None + total_timeout=0.1, on_timeout=None ) async def generator(): for i in range(10): diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 89c1486..fcaafd2 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -3,7 +3,8 @@ __author_email__: str = 'Wolph@wol.ph' __description__: str = ( 'Python Utils is a module with some convenient utilities not included ' - 'with the standard Python install') + 'with the standard Python install' +) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script __version__ = '3.3.1' diff --git a/python_utils/containers.py b/python_utils/containers.py index 00bb86b..cfa717d 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -28,21 +28,17 @@ class CastedDictBase(types.Dict[KT, VT], abc.ABC): _value_cast: VT_cast def __init__( - self, - key_cast: KT_cast = None, - value_cast: VT_cast = None, - *args, - **kwargs + self, + key_cast: KT_cast = None, + value_cast: VT_cast = None, + *args, + **kwargs, ) -> None: self._value_cast = value_cast self._key_cast = key_cast self.update(*args, **kwargs) - def update( - self, - *args: DictUpdateArgs, - **kwargs: VT - ) -> None: + def update(self, *args: DictUpdateArgs, **kwargs: VT) -> None: if args: kwargs.update(*args) diff --git a/python_utils/converters.py b/python_utils/converters.py index ea47d94..2efcbde 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -7,10 +7,10 @@ def to_int( - input_: typing.Optional[str] = None, - default: int = 0, - exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.O[types.Pattern] = None, + input_: typing.Optional[str] = None, + default: int = 0, + exception: types.ExceptionsType = (ValueError, TypeError), + regexp: types.O[types.Pattern] = None, ) -> int: r''' Convert the given input to an integer or return default @@ -97,10 +97,10 @@ def to_int( def to_float( - input_: str, - default: int = 0, - exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.O[types.Pattern] = None, + input_: str, + default: int = 0, + exception: types.ExceptionsType = (ValueError, TypeError), + regexp: types.O[types.Pattern] = None, ) -> types.Number: r''' Convert the given `input_` to an integer or return default @@ -174,9 +174,9 @@ def to_float( def to_unicode( - input_: types.StringTypes, - encoding: str = 'utf-8', - errors: str = 'replace', + input_: types.StringTypes, + encoding: str = 'utf-8', + errors: str = 'replace', ) -> str: '''Convert objects to unicode, if needed decodes string with the given encoding and errors settings. @@ -203,9 +203,9 @@ def to_unicode( def to_str( - input_: types.StringTypes, - encoding: str = 'utf-8', - errors: str = 'replace', + input_: types.StringTypes, + encoding: str = 'utf-8', + errors: str = 'replace', ) -> bytes: '''Convert objects to string, encodes to the given encoding @@ -234,7 +234,8 @@ def to_str( def scale_1024( - x: types.Number, n_prefixes: int, + x: types.Number, + n_prefixes: int, ) -> types.Tuple[types.Number, types.Number]: '''Scale a number down to a suitable size, based on powers of 1024. @@ -262,9 +263,11 @@ def scale_1024( def remap( - value: types.DecimalNumber, - old_min: types.DecimalNumber, old_max: types.DecimalNumber, - new_min: types.DecimalNumber, new_max: types.DecimalNumber, + value: types.DecimalNumber, + old_min: types.DecimalNumber, + old_max: types.DecimalNumber, + new_min: types.DecimalNumber, + new_max: types.DecimalNumber, ) -> types.DecimalNumber: ''' remap a value from one range into another. @@ -340,19 +343,19 @@ def remap( ''' type_: types.Type[types.DecimalNumber] if ( - isinstance(value, decimal.Decimal) or - isinstance(old_min, decimal.Decimal) or - isinstance(old_max, decimal.Decimal) or - isinstance(new_min, decimal.Decimal) or - isinstance(new_max, decimal.Decimal) + isinstance(value, decimal.Decimal) + or isinstance(old_min, decimal.Decimal) + or isinstance(old_max, decimal.Decimal) + or isinstance(new_min, decimal.Decimal) + or isinstance(new_max, decimal.Decimal) ): type_ = decimal.Decimal elif ( - isinstance(value, float) or - isinstance(old_min, float) or - isinstance(old_max, float) or - isinstance(new_min, float) or - isinstance(new_max, float) + isinstance(value, float) + or isinstance(old_min, float) + or isinstance(old_max, float) + or isinstance(new_min, float) + or isinstance(new_max, float) ): type_ = float @@ -369,12 +372,14 @@ def remap( new_range = new_max - new_min # type: ignore if old_range == 0: - raise ValueError('Input range ({}-{}) is empty'.format( - old_min, old_max)) + raise ValueError( + 'Input range ({}-{}) is empty'.format(old_min, old_max) + ) if new_range == 0: - raise ValueError('Output range ({}-{}) is empty'.format( - new_min, new_max)) + raise ValueError( + 'Output range ({}-{}) is empty'.format(new_min, new_max) + ) new_value = (value - old_min) * new_range # type: ignore diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 588dec3..6c237aa 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -111,13 +111,20 @@ def sample(sample_rate: float): Calls to *demo_function* will be limited to 50% approximatly. ''' + def _sample(function): @functools.wraps(function) def __sample(*args, **kwargs): if random.random() < sample_rate: return function(*args, **kwargs) else: - logging.debug('Skipped execution of %r(%r, %r) due to sampling', function, args, kwargs) # noqa: E501 + logging.debug( + 'Skipped execution of %r(%r, %r) due to sampling', + function, + args, + kwargs, + ) # noqa: E501 return __sample + return _sample diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 0e428f8..3449e3e 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -32,7 +32,7 @@ def camel_to_underscore(name: str) -> str: elif i > 3 and not c.isupper(): # Will return the last 3 letters to check if we are changing # case - previous = name[i - 3:i] + previous = name[i - 3 : i] if previous.isalpha() and previous.isupper(): output.insert(len(output) - 1, '_') @@ -77,7 +77,7 @@ def apply_recursive( def timesince( dt: types.Union[datetime.datetime, datetime.timedelta], - default: str = 'just now' + default: str = 'just now', ) -> str: ''' Returns string representing 'time since' e.g. diff --git a/python_utils/import_.py b/python_utils/import_.py index c50da02..b7008ae 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -6,12 +6,12 @@ class DummyException(Exception): def import_global( - name: str, - modules: types.Optional[types.List[str]] = None, - exceptions: types.ExceptionsType = DummyException, - locals_: types.OptionalScope = None, - globals_: types.OptionalScope = None, - level: int = -1, + name: str, + modules: types.Optional[types.List[str]] = None, + exceptions: types.ExceptionsType = DummyException, + locals_: types.OptionalScope = None, + globals_: types.OptionalScope = None, + level: int = -1, ) -> types.Any: '''Import the requested items into the global scope @@ -36,6 +36,7 @@ def import_global( # the current stack if locals_ is None or globals_ is None: import inspect + frame = inspect.stack()[1][0] if locals_ is None: @@ -82,5 +83,13 @@ def import_global( return e finally: # Clean up, just to be sure - del name, name_parts, modules, modules_set, exceptions, locals_, \ - globals_, frame + del ( + name, + name_parts, + modules, + modules_set, + exceptions, + locals_, + globals_, + frame, + ) diff --git a/python_utils/logger.py b/python_utils/logger.py index cbb751c..10ee5e6 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -27,6 +27,7 @@ class LoggerBase(abc.ABC): >>> my_class.exception('exception') >>> my_class.log(0, 'log') ''' + # Being a tad lazy here and not creating a Protocol. # The actual classes define the correct type anyway logger: typing.Any diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 4cf0b4b..85a8099 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -20,8 +20,10 @@ def get_terminal_size() -> typing.Tuple[int, int]: # pragma: no cover try: # Default to 79 characters for IPython notebooks from IPython import get_ipython # type: ignore + ipython = get_ipython() from ipykernel import zmqshell # type: ignore + if isinstance(ipython, zmqshell.ZMQInteractiveShell): return 79, 24 except Exception: # pragma: no cover @@ -31,6 +33,7 @@ def get_terminal_size() -> typing.Tuple[int, int]: # pragma: no cover # This works for Python 3, but not Pypy3. Probably the best method if # it's supported so let's always try import shutil + w, h = shutil.get_terminal_size() if w and h: # The off by one is needed due to progressbars in some cases, for @@ -49,6 +52,7 @@ def get_terminal_size() -> typing.Tuple[int, int]: # pragma: no cover try: import blessings # type: ignore + terminal = blessings.Terminal() w = terminal.width h = terminal.height @@ -100,8 +104,10 @@ def _get_terminal_size_windows(): # pragma: no cover if res: import struct - (_, _, _, _, _, left, top, right, bottom, _, _) = \ - struct.unpack("hhhhHhhhhhh", csbi.raw) + + (_, _, _, _, _, left, top, right, bottom, _, _) = struct.unpack( + "hhhhHhhhhhh", csbi.raw + ) w = right - left h = bottom - top return w, h @@ -113,14 +119,21 @@ def _get_terminal_size_tput(): # pragma: no cover # get terminal width src: http://stackoverflow.com/questions/263890/ try: import subprocess + proc = subprocess.Popen( - ['tput', 'cols'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + ['tput', 'cols'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) output = proc.communicate(input=None) w = int(output[0]) proc = subprocess.Popen( - ['tput', 'lines'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + ['tput', 'lines'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) output = proc.communicate(input=None) h = int(output[0]) return w, h @@ -134,8 +147,10 @@ def ioctl_GWINSZ(fd): import fcntl import termios import struct + size = struct.unpack( - 'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) + 'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234') + ) except Exception: return None return size diff --git a/python_utils/time.py b/python_utils/time.py index 8745c9c..ce1d91c 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -62,7 +62,7 @@ def delta_to_seconds(interval: types.delta_type) -> float: def delta_to_seconds_or_none( - interval: types.Optional[types.delta_type] + interval: types.Optional[types.delta_type], ) -> types.Optional[float]: if interval is None: return None @@ -72,7 +72,7 @@ def delta_to_seconds_or_none( def format_time( timestamp: types.timestamp_type, - precision: datetime.timedelta = datetime.timedelta(seconds=1) + precision: datetime.timedelta = datetime.timedelta(seconds=1), ) -> str: '''Formats timedelta/datetime/seconds @@ -139,8 +139,7 @@ def format_time( def timeout_generator( timeout: types.delta_type, interval: types.delta_type = datetime.timedelta(seconds=1), - iterable: types.Union[types.Iterable, types.Callable] = - itertools.count, + iterable: types.Union[types.Iterable, types.Callable] = itertools.count, interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, ): @@ -203,8 +202,7 @@ def timeout_generator( async def aio_timeout_generator( timeout: types.delta_type, interval: types.delta_type = datetime.timedelta(seconds=1), - iterable: types.Union[ - types.AsyncIterable, types.Callable] = aio.acount, + iterable: types.Union[types.AsyncIterable, types.Callable] = aio.acount, interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, ): @@ -288,11 +286,7 @@ async def aio_generator_timeout_detector( except asyncio.TimeoutError as exception: if on_timeout is not None: await on_timeout( - generator, - timeout, - total_timeout, - exception, - **kwargs + generator, timeout, total_timeout, exception, **kwargs ) break diff --git a/python_utils/types.py b/python_utils/types.py index 282861f..eb3a4f0 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,6 +1,7 @@ import datetime import decimal from typing import * # pragma: no cover + # import * does not import Pattern from typing import Pattern @@ -8,6 +9,7 @@ # support for an optional type shorthand such as `SomeType?` instead of # `Optional[SomeType]`. from typing import Optional as O + # Since the Union operator is only supported for Python 3.10, we'll create a # shorthand for it. from typing import Union as U @@ -39,9 +41,7 @@ 'DecimalNumber', 'delta_type', 'timestamp_type', - # The types from the typing module. - # Super-special typing primitives. 'Annotated', 'Any', @@ -59,7 +59,6 @@ 'Type', 'TypeVar', 'Union', - # ABCs (from collections.abc). 'AbstractSet', # collections.abc.Set. 'ByteString', @@ -85,7 +84,6 @@ 'Collection', 'AsyncGenerator', 'AsyncContextManager', - # Structural checks, a.k.a. protocols. 'Reversible', 'SupportsAbs', @@ -95,7 +93,6 @@ 'SupportsIndex', 'SupportsInt', 'SupportsRound', - # Concrete collection types. 'ChainMap', 'Counter', @@ -109,14 +106,12 @@ 'NamedTuple', # Not really a type. 'TypedDict', # Not really a type. 'Generator', - # Other concrete types. 'BinaryIO', 'IO', 'Match', 'Pattern', 'TextIO', - # One-off things. 'AnyStr', 'cast', diff --git a/setup.py b/setup.py index e2c5962..e82a13d 100644 --- a/setup.py +++ b/setup.py @@ -24,8 +24,9 @@ description=about['__description__'], url=about['__url__'], license='BSD', - packages=setuptools.find_packages(exclude=[ - '_python_utils_tests', '*.__pycache__']), + packages=setuptools.find_packages( + exclude=['_python_utils_tests', '*.__pycache__'] + ), long_description=long_description, tests_require=['pytest'], extras_require={ diff --git a/tox.ini b/tox.ini index 9a3904f..b29ff5b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37, py38, py39, py310, pypy3, flake8, docs +envlist = black, py37, py38, py39, py310, pypy3, flake8, docs skip_missing_interpreters = True [testenv] @@ -14,6 +14,11 @@ setenv = PY_IGNORE_IMPORTMISMATCH=1 deps = -r{toxinidir}/_python_utils_tests/requirements.txt commands = py.test --basetemp="{envtmpdir}" --confcutdir=.. {posargs} python_utils _python_utils_tests +[testenv:black] +basepython = python3 +deps = black +commands = black --skip-string-normalization --line-length 79 {toxinidir}/setup.py {toxinidir}/_python_utils_tests {toxinidir}/python_utils + [testenv:flake8] basepython = python3 deps = flake8 @@ -35,7 +40,7 @@ commands = deps = -r{toxinidir}/docs/requirements.txt [flake8] -ignore = W391, W504, E741 +ignore = W391, W503, E741, E203 exclude = docs From c9d4cd8b091c6aafa7e35eb38746d860dfcb9af1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 May 2022 00:10:23 +0200 Subject: [PATCH 049/132] Fixed bug with batcher skipping items in the case of timeouts thanks to @jorenham --- _python_utils_tests/test_generators.py | 38 ++++++++++++++++++++++++-- python_utils/generators.py | 32 ++++++++++++++-------- 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 25c2853..2a032e0 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -1,3 +1,5 @@ +import asyncio + import pytest import python_utils @@ -16,12 +18,42 @@ async def test_abatcher(): async def test_abatcher_timed(): batches = [] async for batch in python_utils.abatcher( - python_utils.acount(stop=10, delay=0.08), interval=0.2 + python_utils.acount(stop=10, delay=0.08), interval=0.1 ): batches.append(batch) - assert len(batches) == 3 - assert sum(len(batch) for batch in batches) == 10 + assert batches == [[0, 1, 2], [3, 4], [5, 6], [7, 8], [9]] + assert len(batches) == 5 + + +@pytest.mark.asyncio +async def test_abatcher_timed_with_timeout(): + async def generator(): + # Test if the timeout is respected + yield 0 + yield 1 + await asyncio.sleep(0.11) + + # Test if the timeout is respected + yield 2 + yield 3 + await asyncio.sleep(0.11) + + # Test if exceptions are handled correctly + await asyncio.wait_for(asyncio.sleep(1), timeout=0.05) + + # Test if StopAsyncIteration is handled correctly + yield 4 + + batcher = python_utils.abatcher(generator(), interval=0.1) + assert await batcher.__anext__() == [0, 1] + assert await batcher.__anext__() == [2, 3] + + with pytest.raises(asyncio.TimeoutError): + await batcher.__anext__() + + with pytest.raises(StopAsyncIteration): + await batcher.__anext__() def test_batcher(): diff --git a/python_utils/generators.py b/python_utils/generators.py index 3bfc6db..7f0edf7 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -18,27 +18,35 @@ async def abatcher( assert batch_size or interval, 'Must specify either batch_size or interval' + # If interval is specified, use it to determine when to yield the batch + # Alternatively set a really long timeout to keep the code simpler if interval: interval_s = python_utils.delta_to_seconds(interval) - next_yield = time.perf_counter() + interval_s else: - interval_s = 0 - next_yield = 0 + # Set the timeout to 10 years + interval_s = 60 * 60 * 24 * 365 * 10.0 + + next_yield = time.perf_counter() + interval_s + + pending: types.Set = set() while True: try: - if interval_s: - item = await asyncio.wait_for( - generator.__anext__(), interval_s - ) - else: - item = await generator.__anext__() - except (StopAsyncIteration, asyncio.TimeoutError): + done, pending = await asyncio.wait( + pending or [generator.__anext__()], + timeout=interval_s, + return_when=asyncio.FIRST_COMPLETED, + ) + + if done: + for result in done: + batch.append(result.result()) + + except StopAsyncIteration: if batch: yield batch + break - else: - batch.append(item) if batch_size is not None and len(batch) == batch_size: yield batch From 7f0dde9bb9299f2b47636eaa4d0de2c4f7155f33 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 May 2022 00:13:20 +0200 Subject: [PATCH 050/132] Incrementing version to v3.3.2 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index fcaafd2..c379c25 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.3.1' +__version__ = '3.3.2' From d6ade679a5166e3e0f3b949943911bfd948d8cb0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 May 2022 17:25:39 +0200 Subject: [PATCH 051/132] fixed issue with not submitting tasks to `asyncio.wait`. Fixes #35 --- python_utils/generators.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python_utils/generators.py b/python_utils/generators.py index 7f0edf7..ccd8262 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -6,7 +6,7 @@ async def abatcher( - generator: types.AsyncGenerator, + generator: types.AsyncIterator, batch_size: types.Optional[int] = None, interval: types.Optional[types.delta_type] = None, ): @@ -33,7 +33,10 @@ async def abatcher( while True: try: done, pending = await asyncio.wait( - pending or [generator.__anext__()], + pending + or [ + asyncio.create_task(generator.__anext__()), # type: ignore + ], timeout=interval_s, return_when=asyncio.FIRST_COMPLETED, ) From a73192999c6e83cc036eef728e151b9146658a36 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 May 2022 23:17:19 +0200 Subject: [PATCH 052/132] added Python 3.11 testing --- .github/workflows/main.yml | 2 +- tox.ini | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e5f0cdf..b266d6a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 2 strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v2 diff --git a/tox.ini b/tox.ini index b29ff5b..cf4b6d0 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = black, py37, py38, py39, py310, pypy3, flake8, docs +envlist = black, py37, py38, py39, py310, py311, pypy3, flake8, docs skip_missing_interpreters = True [testenv] @@ -8,6 +8,7 @@ basepython = py38: python3.8 py39: python3.9 py310: python3.10 + py311: python3.11 pypy: pypy setenv = PY_IGNORE_IMPORTMISMATCH=1 From b799c8e081ab62e56993321f04cf4e783ac1dcf5 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 May 2022 23:18:57 +0200 Subject: [PATCH 053/132] added Python 3.11 testing --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b266d6a..5a68160 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 2 strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11-dev'] steps: - uses: actions/checkout@v2 From 412cf07ba683a7bddca15fc426b0723cd27beb74 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 May 2022 23:24:05 +0200 Subject: [PATCH 054/132] increased timeout since python 3.11 tests slower --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5a68160..fda01c3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,7 +8,7 @@ on: jobs: build: runs-on: ubuntu-latest - timeout-minutes: 2 + timeout-minutes: 4 strategy: matrix: python-version: ['3.7', '3.8', '3.9', '3.10', '3.11-dev'] From 9520aef32c47d97fc7a77c3ec7064017e99788bc Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 31 May 2022 23:30:36 +0200 Subject: [PATCH 055/132] Incrementing version to v3.3.3 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index c379c25..9125c62 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.3.2' +__version__ = '3.3.3' From 8732cbf79dc03f9c222bc6f91399e9b451c3a113 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 21:09:24 +0200 Subject: [PATCH 056/132] added more type hints, full pyright and mypy support and added py.typed file --- .github/workflows/main.yml | 10 +++++++-- _python_utils_tests/test_time.py | 22 +++++++++++------- pyrightconfig.json | 8 +++++++ python_utils/aio.py | 9 +++++++- python_utils/containers.py | 14 ++++++------ python_utils/converters.py | 4 ++-- python_utils/generators.py | 6 +++-- python_utils/loguru.py | 8 +++---- python_utils/py.typed | 0 python_utils/terminal.py | 38 ++++++++++++++++++-------------- python_utils/types.py | 6 ++--- setup.cfg | 7 ++++++ tox.ini | 29 +++++++++++++++++------- 13 files changed, 107 insertions(+), 54 deletions(-) create mode 100644 pyrightconfig.json create mode 100644 python_utils/py.typed diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fda01c3..a98f818 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,7 +32,7 @@ jobs: - name: pytest run: py.test - docs: + docs_and_lint: runs-on: ubuntu-latest timeout-minutes: 2 steps: @@ -44,7 +44,13 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools - pip install -e '.[docs]' + pip install -e '.[docs,tests]' pyright flake8 mypy - name: build docs run: make html working-directory: docs/ + - name: flake8 + run: flake8 -v python_utils setup.py + - name: mypy + run: mypy python_utils setup.py + - name: pyright + run: pyright diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 3da6284..337d4da 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -71,6 +71,9 @@ def test_timeout_generator( @pytest.mark.asyncio async def test_aio_generator_timeout_detector(): + # Make pyright happy + i = None + async def generator(): for i in range(10): await asyncio.sleep(i / 100.0) @@ -106,52 +109,55 @@ async def generator(): @pytest.mark.asyncio async def test_aio_generator_timeout_detector_decorator(): + # Make pyright happy + i = None + # Test regular timeout with reraise @python_utils.aio_generator_timeout_detector_decorator(timeout=0.05) - async def generator(): + async def generator_timeout(): for i in range(10): await asyncio.sleep(i / 100.0) yield i with pytest.raises(asyncio.TimeoutError): - async for i in generator(): + async for i in generator_timeout(): pass # Test regular timeout with clean exit @python_utils.aio_generator_timeout_detector_decorator( timeout=0.05, on_timeout=None ) - async def generator(): + async def generator_clean(): for i in range(10): await asyncio.sleep(i / 100.0) yield i - async for i in generator(): + async for i in generator_clean(): pass assert i == 4 # Test total timeout with reraise @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.1) - async def generator(): + async def generator_reraise(): for i in range(10): await asyncio.sleep(i / 100.0) yield i with pytest.raises(asyncio.TimeoutError): - async for i in generator(): + async for i in generator_reraise(): pass # Test total timeout with clean exit @python_utils.aio_generator_timeout_detector_decorator( total_timeout=0.1, on_timeout=None ) - async def generator(): + async def generator_clean_total(): for i in range(10): await asyncio.sleep(i / 100.0) yield i - async for i in generator(): + async for i in generator_clean_total(): pass assert i == 4 diff --git a/pyrightconfig.json b/pyrightconfig.json new file mode 100644 index 0000000..aaf3faf --- /dev/null +++ b/pyrightconfig.json @@ -0,0 +1,8 @@ +{ + "include": [ + "python_utils" + ], + "exclude": [ + "python_utils/types.py", + ], +} diff --git a/python_utils/aio.py b/python_utils/aio.py index d099af6..de1ff87 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -5,8 +5,15 @@ import asyncio import itertools +from . import types -async def acount(start=0, step=1, delay=0, stop=None): + +async def acount( + start: float = 0, + step: float = 1, + delay: float = 0, + stop: types.Optional[float] = None, +) -> types.AsyncIterator[float]: '''Asyncio version of itertools.count()''' for item in itertools.count(start, step): # pragma: no branch if stop is not None and item >= stop: diff --git a/python_utils/containers.py b/python_utils/containers.py index cfa717d..a51bfc6 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -31,8 +31,8 @@ def __init__( self, key_cast: KT_cast = None, value_cast: VT_cast = None, - *args, - **kwargs, + *args: DictUpdateArgs, + **kwargs: VT, ) -> None: self._value_cast = value_cast self._key_cast = key_cast @@ -53,7 +53,7 @@ def __setitem__(self, key: Any, value: Any) -> None: return super().__setitem__(key, value) -class CastedDict(CastedDictBase): +class CastedDict(CastedDictBase[KT, VT]): ''' Custom dictionary that casts keys and values to the specified typing. @@ -88,14 +88,14 @@ class CastedDict(CastedDictBase): {1: 2, '3': '4', '5': '6', '7': '8'} ''' - def __setitem__(self, key, value): + def __setitem__(self, key: Any, value: Any) -> None: if self._value_cast is not None: value = self._value_cast(value) super().__setitem__(key, value) -class LazyCastedDict(CastedDictBase): +class LazyCastedDict(CastedDictBase[KT, VT]): ''' Custom dictionary that casts keys and lazily casts values to the specified typing. Note that the values are cast only when they are accessed and @@ -141,13 +141,13 @@ class LazyCastedDict(CastedDictBase): '4' ''' - def __setitem__(self, key, value): + def __setitem__(self, key: Any, value: Any) -> None: if self._key_cast is not None: key = self._key_cast(key) super().__setitem__(key, value) - def __getitem__(self, key) -> VT: + def __getitem__(self, key: Any) -> VT: if self._key_cast is not None: key = self._key_cast(key) diff --git a/python_utils/converters.py b/python_utils/converters.py index 2efcbde..c1eba73 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -10,7 +10,7 @@ def to_int( input_: typing.Optional[str] = None, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.O[types.Pattern] = None, + regexp: types.Optional[types.Pattern] = None, ) -> int: r''' Convert the given input to an integer or return default @@ -100,7 +100,7 @@ def to_float( input_: str, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.O[types.Pattern] = None, + regexp: types.Optional[types.Pattern] = None, ) -> types.Number: r''' Convert the given `input_` to an integer or return default diff --git a/python_utils/generators.py b/python_utils/generators.py index ccd8262..2930334 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -9,7 +9,7 @@ async def abatcher( generator: types.AsyncIterator, batch_size: types.Optional[int] = None, interval: types.Optional[types.delta_type] = None, -): +) -> types.AsyncIterator[list]: ''' Asyncio generator wrapper that returns items with a given batch size or interval (whichever is reached first). @@ -64,7 +64,9 @@ async def abatcher( next_yield = time.perf_counter() + interval_s -def batcher(iterable, batch_size): +def batcher( + iterable: types.Iterable, batch_size: int = 10 +) -> types.Iterator[list]: ''' Generator wrapper that returns items with a given batch size ''' diff --git a/python_utils/loguru.py b/python_utils/loguru.py index 0bf5040..7f172b6 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -1,15 +1,15 @@ from __future__ import annotations -from . import logger - import loguru +from . import logger as logger_module + __all__ = ['Logurud'] -class Logurud(logger.LoggerBase): +class Logurud(logger_module.LoggerBase): logger: loguru.Logger def __new__(cls, *args, **kwargs): - cls.logger: loguru.Loguru = loguru.logger.opt(depth=1) + cls.logger: loguru.Logger = loguru.logger.opt(depth=1) return super().__new__(cls) diff --git a/python_utils/py.typed b/python_utils/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 85a8099..19c494d 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -3,8 +3,11 @@ from . import converters +Dimensions = typing.Tuple[int, int] +OptionalDimensions = typing.Optional[Dimensions] -def get_terminal_size() -> typing.Tuple[int, int]: # pragma: no cover + +def get_terminal_size() -> Dimensions: # pragma: no cover '''Get the current size of your terminal Multiple returns are not always a good idea, but in this case it greatly @@ -62,35 +65,36 @@ def get_terminal_size() -> typing.Tuple[int, int]: # pragma: no cover pass try: - w, h = _get_terminal_size_linux() - if w and h: - return w, h + # The method can return None so we don't unpack it + wh = _get_terminal_size_linux() + if wh is not None and all(wh): + return wh except Exception: # pragma: no cover pass try: # Windows detection doesn't always work, let's try anyhow - w, h = _get_terminal_size_windows() - if w and h: - return w, h + wh = _get_terminal_size_windows() + if wh is not None and all(wh): + return wh except Exception: # pragma: no cover pass try: # needed for window's python in cygwin's xterm! - w, h = _get_terminal_size_tput() - if w and h: - return w, h + wh = _get_terminal_size_tput() + if wh is not None and all(wh): + return wh except Exception: # pragma: no cover pass return 79, 24 -def _get_terminal_size_windows(): # pragma: no cover +def _get_terminal_size_windows() -> OptionalDimensions: # pragma: no cover res = None try: - from ctypes import windll, create_string_buffer + from ctypes import windll, create_string_buffer # type: ignore # stdin handle is -10 # stdout handle is -11 @@ -115,7 +119,7 @@ def _get_terminal_size_windows(): # pragma: no cover return None -def _get_terminal_size_tput(): # pragma: no cover +def _get_terminal_size_tput() -> OptionalDimensions: # pragma: no cover # get terminal width src: http://stackoverflow.com/questions/263890/ try: import subprocess @@ -141,19 +145,19 @@ def _get_terminal_size_tput(): # pragma: no cover return None -def _get_terminal_size_linux(): # pragma: no cover +def _get_terminal_size_linux() -> OptionalDimensions: # pragma: no cover def ioctl_GWINSZ(fd): try: import fcntl import termios import struct - size = struct.unpack( - 'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234') + return struct.unpack( + 'hh', + fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'), # type: ignore ) except Exception: return None - return size size = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) diff --git a/python_utils/types.py b/python_utils/types.py index eb3a4f0..23acf3d 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,6 +1,6 @@ import datetime import decimal -from typing import * # pragma: no cover +from typing import * # type: ignore # pragma: no cover # import * does not import Pattern from typing import Pattern @@ -8,11 +8,11 @@ # Quickhand for optional because it gets so much use. If only Python had # support for an optional type shorthand such as `SomeType?` instead of # `Optional[SomeType]`. -from typing import Optional as O +from typing import Optional as O # noqa # Since the Union operator is only supported for Python 3.10, we'll create a # shorthand for it. -from typing import Union as U +from typing import Union as U # noqa Scope = Dict[str, Any] OptionalScope = O[Scope] diff --git a/setup.cfg b/setup.cfg index cc55116..9cad36b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,4 +32,11 @@ sign = 1 [flake8] per-file-ignores = python_utils/types.py: F403,F405 +ignore = W391, W503, E741, E203 exclude = + docs + +[mypy] +files = + python_utils, + _python_utils_tests diff --git a/tox.ini b/tox.ini index cf4b6d0..29ab352 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = black, py37, py38, py39, py310, py311, pypy3, flake8, docs +envlist = black, py37, py38, py39, py310, py311, pypy3, flake8, docs, mypy, pyright skip_missing_interpreters = True [testenv] @@ -12,8 +12,14 @@ basepython = pypy: pypy setenv = PY_IGNORE_IMPORTMISMATCH=1 -deps = -r{toxinidir}/_python_utils_tests/requirements.txt -commands = py.test --basetemp="{envtmpdir}" --confcutdir=.. {posargs} python_utils _python_utils_tests +deps = + mypy + pyright + -r{toxinidir}/_python_utils_tests/requirements.txt +commands = + mypy + pyright + py.test --basetemp="{envtmpdir}" --confcutdir=.. {posargs} python_utils _python_utils_tests [testenv:black] basepython = python3 @@ -25,6 +31,18 @@ basepython = python3 deps = flake8 commands = flake8 python_utils {posargs} +[testenv:pyright] +basepython = python3 +deps = + pyright + -r{toxinidir}/_python_utils_tests/requirements.txt +commands = pyright {posargs} + +[testenv:mypy] +basepython = python3 +deps = -r{toxinidir}/_python_utils_tests/requirements.txt +commands = mypy {posargs} + [testenv:docs] basepython = python3 whitelist_externals = @@ -40,8 +58,3 @@ commands = sphinx-build -n -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} deps = -r{toxinidir}/docs/requirements.txt -[flake8] -ignore = W391, W503, E741, E203 -exclude = - docs - From 8f28de5af171cb4ef7f92cb03060ea059aff92a2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 21:12:00 +0200 Subject: [PATCH 057/132] updated github actions workflow --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a98f818..b7d08dc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,9 +14,9 @@ jobs: python-version: ['3.7', '3.8', '3.9', '3.10', '3.11-dev'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -36,9 +36,9 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 2 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: '3.10' - name: Install dependencies From 3d1a09a387156163de32a1474c91348ef494a807 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 21:12:19 +0200 Subject: [PATCH 058/132] Incrementing version to v3.4.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 9125c62..1c7ada8 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.3.3' +__version__ = '3.4.0' From 3345895613e2147738fd4addb8ffaf59a12eb22e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 22:47:43 +0200 Subject: [PATCH 059/132] Added py.typed --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index fc8acef..0680580 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,5 +7,6 @@ include requirements.txt include setup.cfg include setup.py include tox.ini +include python_utils/py.typed recursive-include _python_utils_tests *.py *.txt recursive-exclude __pycache__ * From ef29fc95b637da0e2e1c977f6550ab87fb73bf37 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 22:47:52 +0200 Subject: [PATCH 060/132] Incrementing version to v3.4.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 1c7ada8..59b4da2 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.4.0' +__version__ = '3.4.1' From 8cc768c978297f0d6664347ad54fde1efa638d9b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 22:48:25 +0200 Subject: [PATCH 061/132] Incrementing version to v3.4.2 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 59b4da2..b548efc 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.4.1' +__version__ = '3.4.2' From 978e4f176c4dd366a6f9ea8d9589d1d2c9eb349f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 22:48:43 +0200 Subject: [PATCH 062/132] Incrementing version to v3.4.3 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index b548efc..f1115b3 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.4.2' +__version__ = '3.4.3' From 18f96e71564199351cb83deaa279ffd8b36d5417 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 23:17:40 +0200 Subject: [PATCH 063/132] added py.typed --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e82a13d..f39a711 100644 --- a/setup.py +++ b/setup.py @@ -25,8 +25,9 @@ url=about['__url__'], license='BSD', packages=setuptools.find_packages( - exclude=['_python_utils_tests', '*.__pycache__'] + exclude=['_python_utils_tests', '*.__pycache__'], ), + package_data={'python_utils': ['py.typed']}, long_description=long_description, tests_require=['pytest'], extras_require={ From f3f1bfd0d306e3484f14cfbfa6412cb908e9b4f1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 23:17:50 +0200 Subject: [PATCH 064/132] Incrementing version to v3.4.4 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index f1115b3..a78a4e1 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.4.3' +__version__ = '3.4.4' From c7440f2af602e7385427c7a78857b0488610096c Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 23:40:19 +0200 Subject: [PATCH 065/132] added traceback type --- python_utils/types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python_utils/types.py b/python_utils/types.py index 23acf3d..fa5d202 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -132,4 +132,5 @@ 'TYPE_CHECKING', 'TypeAlias', 'TypeGuard', + 'TracebackType', ] From 0bc8a741d09946b7310ba3ac083b14a610cd219d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 29 Oct 2022 23:40:45 +0200 Subject: [PATCH 066/132] Incrementing version to v3.4.5 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index a78a4e1..84738de 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.4.4' +__version__ = '3.4.5' From dd2760add25931e294fccace2d214ab6d482df4b Mon Sep 17 00:00:00 2001 From: LGTM Migrator Date: Thu, 10 Nov 2022 22:00:23 +0000 Subject: [PATCH 067/132] Add CodeQL workflow for GitHub code scanning --- .github/workflows/codeql.yml | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..da17bb3 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,41 @@ +name: "CodeQL" + +on: + push: + branches: [ "develop" ] + pull_request: + branches: [ "develop" ] + schedule: + - cron: "46 1 * * 3" + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ python ] + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + queries: +security-and-quality + + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{ matrix.language }}" From 3c722f35e0d65dcdfa36e61918038cfcc3ef8b7f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 02:37:17 +0100 Subject: [PATCH 068/132] Added unique list container --- .coveragerc | 1 + _python_utils_tests/test_containers.py | 29 +++++++ python_utils/containers.py | 114 +++++++++++++++++++++++++ 3 files changed, 144 insertions(+) create mode 100644 _python_utils_tests/test_containers.py diff --git a/.coveragerc b/.coveragerc index 1e4e2d1..ec82fd8 100644 --- a/.coveragerc +++ b/.coveragerc @@ -22,3 +22,4 @@ exclude_lines = if 0: if __name__ == .__main__.: if typing.TYPE_CHECKING: + if types.TYPE_CHECKING: diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py new file mode 100644 index 0000000..ba511ae --- /dev/null +++ b/_python_utils_tests/test_containers.py @@ -0,0 +1,29 @@ +import pytest + +from python_utils import containers + + +def test_unique_list_ignore(): + a = containers.UniqueList() + a.append(1) + a.append(1) + assert a == [1] + + a = containers.UniqueList(*range(20)) + with pytest.raises(RuntimeError): + a[10:20:2] = [1, 2, 3, 4, 5] + + a[3] = 5 + + +def test_unique_list_raise(): + a = containers.UniqueList(*range(20), on_duplicate='raise') + with pytest.raises(ValueError): + a[10:20:2] = [1, 2, 3, 4, 5] + + a[10:20:2] = [21, 22, 23, 24, 25] + with pytest.raises(ValueError): + a[3] = 5 + + del a[10] + del a[5:15] diff --git a/python_utils/containers.py b/python_utils/containers.py index a51bfc6..d5b00ab 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -22,6 +22,8 @@ '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] +OnDuplicate = types.Literal['raise', 'ignore'] + class CastedDictBase(types.Dict[KT, VT], abc.ABC): _key_cast: KT_cast @@ -173,6 +175,118 @@ def values(self) -> Generator[VT, None, None]: # type: ignore yield self._value_cast(value) +class UniqueList(types.List[VT]): + ''' + A list that only allows unique values. Duplicate values are ignored by + default, but can be configured to raise an exception instead. + + >>> l = UniqueList(1, 2, 3) + >>> l.append(4) + >>> l.append(4) + >>> l.insert(0, 4) + >>> l.insert(0, 5) + >>> l[1] = 10 + >>> l + [5, 10, 2, 3, 4] + + >>> l = UniqueList(1, 2, 3, on_duplicate='raise') + >>> l.append(4) + >>> l.append(4) + Traceback (most recent call last): + ... + ValueError: Duplicate value: 4 + >>> l.insert(0, 4) + Traceback (most recent call last): + ... + ValueError: Duplicate value: 4 + >>> 4 in l + True + >>> l[0] + 1 + >>> l[1] = 4 + Traceback (most recent call last): + ... + ValueError: Duplicate value: 4 + ''' + _set: set[VT] + + def __init__(self, *args: VT, on_duplicate: OnDuplicate = 'ignore'): + self.on_duplicate = on_duplicate + self._set = set() + super().__init__() + for arg in args: + self.append(arg) + + def insert(self, index: types.SupportsIndex, value: VT) -> None: + if value in self._set: + if self.on_duplicate == 'raise': + raise ValueError('Duplicate value: %s' % value) + else: + return + + self._set.add(value) + super().insert(index, value) + + def append(self, value: VT) -> None: + if value in self._set: + if self.on_duplicate == 'raise': + raise ValueError('Duplicate value: %s' % value) + else: + return + + self._set.add(value) + super().append(value) + + def __contains__(self, item): + return item in self._set + + @types.overload + @abc.abstractmethod + def __setitem__(self, index: types.SupportsIndex, value: VT) -> None: + ... + + @types.overload + @abc.abstractmethod + def __setitem__(self, index: slice, value: types.Iterable[VT]) -> None: + ... + + def __setitem__(self, indices, values) -> None: + if isinstance(indices, slice): + if self.on_duplicate == 'ignore': + raise RuntimeError( + 'ignore mode while setting slices introduces ambiguous ' + 'behaviour and is therefore not supported' + ) + + duplicates = set(values) & self._set + if duplicates and values != self[indices]: + raise ValueError('Duplicate values: %s' % duplicates) + + self._set.update(values) + super().__setitem__(indices, values) + else: + if values in self._set and values != self[indices]: + if self.on_duplicate == 'raise': + raise ValueError('Duplicate value: %s' % values) + else: + return + + self._set.add(values) + super().__setitem__(indices, values) + + def __delitem__( + self, + index: types.Union[types.SupportsIndex, slice] + ) -> None: + if isinstance(index, slice): + for value in self[index]: + self._set.remove(value) + else: + self._set.remove(self[index]) + + super().__delitem__(index) + + if __name__ == '__main__': import doctest From 60d6d16e367b99b41a45bb7c9f81277a28e35019 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 03:10:14 +0100 Subject: [PATCH 069/132] python 3.7 type compatibility --- python_utils/containers.py | 8 ++++---- python_utils/types.py | 10 +++++++++- setup.py | 3 +++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index d5b00ab..8070928 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -22,7 +22,7 @@ '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] -OnDuplicate = types.Literal['raise', 'ignore'] +OnDuplicate: types.Literal['raise', 'ignore'] class CastedDictBase(types.Dict[KT, VT], abc.ABC): @@ -208,6 +208,7 @@ class UniqueList(types.List[VT]): ... ValueError: Duplicate value: 4 ''' + _set: set[VT] def __init__(self, *args: VT, on_duplicate: OnDuplicate = 'ignore'): @@ -275,9 +276,8 @@ def __setitem__(self, indices, values) -> None: super().__setitem__(indices, values) def __delitem__( - self, - index: types.Union[types.SupportsIndex, slice] - ) -> None: + self, index: types.Union[types.SupportsIndex, slice] + ) -> None: if isinstance(index, slice): for value in self[index]: self._set.remove(value) diff --git a/python_utils/types.py b/python_utils/types.py index fa5d202..2b89475 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -5,6 +5,11 @@ # import * does not import Pattern from typing import Pattern +try: + from typing import Literal, SupportsIndex # type: ignore +except ImportError: + from typing_extensions import Literal, SupportsIndex + # Quickhand for optional because it gets so much use. If only Python had # support for an optional type shorthand such as `SomeType?` instead of # `Optional[SomeType]`. @@ -33,7 +38,9 @@ None, ] -assert Pattern +assert Pattern # type: ignore +assert Literal +assert SupportsIndex __all__ = [ 'OptionalScope', @@ -52,6 +59,7 @@ 'ForwardRef', 'Generic', 'Literal', + 'SupportsIndex', 'Optional', 'ParamSpec', 'Protocol', diff --git a/setup.py b/setup.py index f39a711..3096e40 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,8 @@ else: long_description = 'See http://pypi.python.org/pypi/python-utils/' +install_requires = [] + if __name__ == '__main__': setuptools.setup( python_requires='>3.6.0', @@ -29,6 +31,7 @@ ), package_data={'python_utils': ['py.typed']}, long_description=long_description, + install_requires=['typing_extensions;python_version<"3.8"'], tests_require=['pytest'], extras_require={ 'loguru': [ From ef34212af44befba0c4be36513ec4bc2721c786b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 03:38:17 +0100 Subject: [PATCH 070/132] flake8 doesn't understand @typing.overload --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 9cad36b..ca2401e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,7 @@ sign = 1 [flake8] per-file-ignores = python_utils/types.py: F403,F405 -ignore = W391, W503, E741, E203 +ignore = W391, W503, E741, E203, F811 exclude = docs From ace90a5398515f16dada25cc11918c0c3284f77d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 04:05:49 +0100 Subject: [PATCH 071/132] finally done with type hinting issues between python versions? --- python_utils/containers.py | 8 +++++--- python_utils/types.py | 13 +++++++------ setup.py | 2 -- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index 8070928..2963cf7 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -22,8 +22,6 @@ '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] -OnDuplicate: types.Literal['raise', 'ignore'] - class CastedDictBase(types.Dict[KT, VT], abc.ABC): _key_cast: KT_cast @@ -211,7 +209,11 @@ class UniqueList(types.List[VT]): _set: set[VT] - def __init__(self, *args: VT, on_duplicate: OnDuplicate = 'ignore'): + def __init__( + self, + *args: VT, + on_duplicate: types.Literal['raise', 'ignore'] = 'ignore' + ): self.on_duplicate = on_duplicate self._set = set() super().__init__() diff --git a/python_utils/types.py b/python_utils/types.py index 2b89475..1e749c8 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,13 +1,14 @@ import datetime import decimal +import sys from typing import * # type: ignore # pragma: no cover # import * does not import Pattern from typing import Pattern -try: - from typing import Literal, SupportsIndex # type: ignore -except ImportError: +if sys.version_info >= (3, 8): # pragma: no cover + from typing import Literal, SupportsIndex +else: # pragma: no cover from typing_extensions import Literal, SupportsIndex # Quickhand for optional because it gets so much use. If only Python had @@ -38,9 +39,9 @@ None, ] -assert Pattern # type: ignore -assert Literal -assert SupportsIndex +assert Pattern is not None # type: ignore +assert Literal is not None +assert SupportsIndex is not None __all__ = [ 'OptionalScope', diff --git a/setup.py b/setup.py index 3096e40..af90617 100644 --- a/setup.py +++ b/setup.py @@ -14,8 +14,6 @@ else: long_description = 'See http://pypi.python.org/pypi/python-utils/' -install_requires = [] - if __name__ == '__main__': setuptools.setup( python_requires='>3.6.0', From 50ec301aa859e1e269e24aa9b32e5bafe9a6bba2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 17:24:29 +0100 Subject: [PATCH 072/132] Fixed pyright tests --- python_utils/containers.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index 2963cf7..f5b2c02 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -212,7 +212,7 @@ class UniqueList(types.List[VT]): def __init__( self, *args: VT, - on_duplicate: types.Literal['raise', 'ignore'] = 'ignore' + on_duplicate: types.Literal['raise', 'ignore'] = 'ignore', ): self.on_duplicate = on_duplicate self._set = set() @@ -244,13 +244,11 @@ def __contains__(self, item): return item in self._set @types.overload - @abc.abstractmethod - def __setitem__(self, index: types.SupportsIndex, value: VT) -> None: + def __setitem__(self, indices: types.SupportsIndex, values: VT) -> None: ... @types.overload - @abc.abstractmethod - def __setitem__(self, index: slice, value: types.Iterable[VT]) -> None: + def __setitem__(self, indices: slice, values: types.Iterable[VT]) -> None: ... def __setitem__(self, indices, values) -> None: From a7d5d843baf55c7ee944192497979a55dfe391e3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 17:38:15 +0100 Subject: [PATCH 073/132] Make coverage happy as well --- .coveragerc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.coveragerc b/.coveragerc index ec82fd8..61d9675 100644 --- a/.coveragerc +++ b/.coveragerc @@ -23,3 +23,6 @@ exclude_lines = if __name__ == .__main__.: if typing.TYPE_CHECKING: if types.TYPE_CHECKING: + @overload + @types.overload + @typing.overload From d71b9b74483d8020bf1a6ba47fb5e522d2e79977 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 17:44:11 +0100 Subject: [PATCH 074/132] Incrementing version to v3.5.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 84738de..77f1363 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.4.5' +__version__ = '3.5.0' From 814a080061bacb3698a499b13fad63b24cd7c59a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 23:08:46 +0100 Subject: [PATCH 075/132] Improved type hinting thanks to @jorenham --- python_utils/containers.py | 15 ++++++++------- tox.ini | 6 +++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index f5b2c02..68d48c8 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -14,6 +14,7 @@ DT = types.Dict[KT, VT] KT_cast = types.Optional[types.Callable[[Any], KT]] VT_cast = types.Optional[types.Callable[[Any], VT]] +HT = types.TypeVar('HT', bound=types.Hashable) # Using types.Union instead of | since Python 3.7 doesn't fully support it DictUpdateArgs = types.Union[ @@ -173,7 +174,7 @@ def values(self) -> Generator[VT, None, None]: # type: ignore yield self._value_cast(value) -class UniqueList(types.List[VT]): +class UniqueList(types.List[HT]): ''' A list that only allows unique values. Duplicate values are ignored by default, but can be configured to raise an exception instead. @@ -207,11 +208,11 @@ class UniqueList(types.List[VT]): ValueError: Duplicate value: 4 ''' - _set: set[VT] + _set: set[HT] def __init__( self, - *args: VT, + *args: HT, on_duplicate: types.Literal['raise', 'ignore'] = 'ignore', ): self.on_duplicate = on_duplicate @@ -220,7 +221,7 @@ def __init__( for arg in args: self.append(arg) - def insert(self, index: types.SupportsIndex, value: VT) -> None: + def insert(self, index: types.SupportsIndex, value: HT) -> None: if value in self._set: if self.on_duplicate == 'raise': raise ValueError('Duplicate value: %s' % value) @@ -230,7 +231,7 @@ def insert(self, index: types.SupportsIndex, value: VT) -> None: self._set.add(value) super().insert(index, value) - def append(self, value: VT) -> None: + def append(self, value: HT) -> None: if value in self._set: if self.on_duplicate == 'raise': raise ValueError('Duplicate value: %s' % value) @@ -244,11 +245,11 @@ def __contains__(self, item): return item in self._set @types.overload - def __setitem__(self, indices: types.SupportsIndex, values: VT) -> None: + def __setitem__(self, indices: types.SupportsIndex, values: HT) -> None: ... @types.overload - def __setitem__(self, indices: slice, values: types.Iterable[VT]) -> None: + def __setitem__(self, indices: slice, values: types.Iterable[HT]) -> None: ... def __setitem__(self, indices, values) -> None: diff --git a/tox.ini b/tox.ini index 29ab352..901cfce 100644 --- a/tox.ini +++ b/tox.ini @@ -44,7 +44,12 @@ deps = -r{toxinidir}/_python_utils_tests/requirements.txt commands = mypy {posargs} [testenv:docs] +changedir = basepython = python3 +deps = -r{toxinidir}/docs/requirements.txt +allowlist_externals = + rm + mkdir whitelist_externals = rm cd @@ -56,5 +61,4 @@ commands = sphinx-apidoc -o docs/ python_utils rm -f docs/modules.rst sphinx-build -n -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} -deps = -r{toxinidir}/docs/requirements.txt From 544d13e2a0d57960f637ba1d476831a3cf770d83 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 8 Feb 2023 23:13:58 +0100 Subject: [PATCH 076/132] Incrementing version to v3.5.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 77f1363..0d5de66 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.5.0' +__version__ = '3.5.1' From 1398a93521f82bf615c29bdb44ea0018681ebaf2 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 9 Feb 2023 15:32:50 +0100 Subject: [PATCH 077/132] Attempting type hinting improvements --- python_utils/__init__.py | 4 ++++ python_utils/containers.py | 18 ++++++++++++------ tox.ini | 2 +- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/python_utils/__init__.py b/python_utils/__init__.py index de72a02..900bcc8 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -12,6 +12,7 @@ types, ) from .aio import acount +from .containers import CastedDict, LazyCastedDict, UniqueList from .converters import remap, scale_1024, to_float, to_int, to_str, to_unicode from .decorators import listify, set_attributes from .exceptions import raise_exception, reraise @@ -70,4 +71,7 @@ 'raise_exception', 'Logged', 'LoggerBase', + 'CastedDict', + 'LazyCastedDict', + 'UniqueList', ] diff --git a/python_utils/containers.py b/python_utils/containers.py index 68d48c8..f8920a8 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import abc import typing from typing import Any, Generator @@ -9,11 +7,17 @@ if typing.TYPE_CHECKING: import _typeshed # noqa: F401 +#: A type alias for a type that can be used as a key in a dictionary. KT = types.TypeVar('KT') +#: A type alias for a type that can be used as a value in a dictionary. VT = types.TypeVar('VT') +#: A type alias for a dictionary with keys of type KT and values of type VT. DT = types.Dict[KT, VT] +#: A type alias for the casted type of a dictionary key. KT_cast = types.Optional[types.Callable[[Any], KT]] +#: A type alias for the casted type of a dictionary value. VT_cast = types.Optional[types.Callable[[Any], VT]] +#: A type alias for the hashable values of the `UniqueList` HT = types.TypeVar('HT', bound=types.Hashable) # Using types.Union instead of | since Python 3.7 doesn't fully support it @@ -61,7 +65,7 @@ class CastedDict(CastedDictBase[KT, VT]): Note that you can specify the types for mypy and type hinting with: CastedDict[int, int](int, int) - >>> d = CastedDict(int, int) + >>> d: CastedDict[int, int] = CastedDict(int, int) >>> d[1] = 2 >>> d['3'] = '4' >>> d.update({'5': '6'}) @@ -105,7 +109,7 @@ class LazyCastedDict(CastedDictBase[KT, VT]): Note that you can specify the types for mypy and type hinting with: LazyCastedDict[int, int](int, int) - >>> d = LazyCastedDict(int, int) + >>> d: LazyCastedDict[int, int] = LazyCastedDict(int, int) >>> d[1] = 2 >>> d['3'] = '4' >>> d.update({'5': '6'}) @@ -159,7 +163,9 @@ def __getitem__(self, key: Any) -> VT: return value - def items(self) -> Generator[tuple[KT, VT], None, None]: # type: ignore + def items( # type: ignore + self, + ) -> Generator[types.Tuple[KT, VT], None, None]: if self._value_cast is None: yield from super().items() else: @@ -208,7 +214,7 @@ class UniqueList(types.List[HT]): ValueError: Duplicate value: 4 ''' - _set: set[HT] + _set: types.Set[HT] def __init__( self, diff --git a/tox.ini b/tox.ini index 901cfce..8290500 100644 --- a/tox.ini +++ b/tox.ini @@ -60,5 +60,5 @@ commands = mkdir -p docs/_static sphinx-apidoc -o docs/ python_utils rm -f docs/modules.rst - sphinx-build -n -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} + sphinx-build -W -b html -d docs/_build/doctrees docs docs/_build/html {posargs} From e20454482ea64a2010cfd4f7c6f09cdb61c4a16f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 9 Feb 2023 15:40:50 +0100 Subject: [PATCH 078/132] Incrementing version to v3.5.2 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 0d5de66..7e57bd8 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.5.1' +__version__ = '3.5.2' From 2df4ac04fb6e01c8581626525cfec62db69d7463 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 9 Feb 2023 16:14:05 +0100 Subject: [PATCH 079/132] Fixed last issue of #37 --- python_utils/loguru.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python_utils/loguru.py b/python_utils/loguru.py index 7f172b6..c09fd4f 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -2,7 +2,7 @@ import loguru -from . import logger as logger_module +from . import logger as logger_module, types __all__ = ['Logurud'] @@ -10,6 +10,6 @@ class Logurud(logger_module.LoggerBase): logger: loguru.Logger - def __new__(cls, *args, **kwargs): + def __new__(cls, *args: types.Any, **kwargs: types.Any) -> Logurud: cls.logger: loguru.Logger = loguru.logger.opt(depth=1) return super().__new__(cls) From 0ad935e2b6e30c6328ec97b9a698e87a065f71ba Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 13 Mar 2023 22:12:57 +0100 Subject: [PATCH 080/132] added security contact information --- README.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.rst b/README.rst index 9924421..9573a7e 100644 --- a/README.rst +++ b/README.rst @@ -25,6 +25,13 @@ Links - Documentation: https://python-utils.readthedocs.io/en/latest/ - My blog: https://wol.ph/ +Security contact information +------------------------------------------------------------------------------ + +To report a security vulnerability, please use the +`Tidelift security contact `_. +Tidelift will coordinate the fix and disclosure. + Requirements for installing: ------------------------------------------------------------------------------ From 94273313cfa0229708966287e42d8eaa2e26cbdc Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 21 May 2023 01:10:16 +0200 Subject: [PATCH 081/132] Applied black formatting --- docs/conf.py | 2 +- python_utils/formatters.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 9d82d43..63e2a44 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,6 +13,7 @@ from datetime import date import os import sys + sys.path.insert(0, os.path.abspath('..')) from python_utils import __about__ @@ -63,4 +64,3 @@ # html_static_path = ['_static'] intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} - diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 3449e3e..d2d76cb 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -44,7 +44,7 @@ def camel_to_underscore(name: str) -> str: def apply_recursive( function: types.Callable[[str], str], data: types.OptionalScope = None, - **kwargs + **kwargs, ) -> types.OptionalScope: ''' Apply a function to all keys in a scope recursively From 5a200d87dec79ed263db2983b43ff834021be1cd Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 25 May 2023 17:16:42 +0200 Subject: [PATCH 082/132] Dropped Python 3.7 support and added full generic strict Pyright and Mypy compliant type hinting --- .gitignore | 1 + pyrightconfig.json | 8 ----- python_utils/aio.py | 12 ++++--- python_utils/containers.py | 55 +++++++++++++++++------------ python_utils/converters.py | 54 ++++++++++++++--------------- python_utils/decorators.py | 53 ++++++++++++++++++++-------- python_utils/exceptions.py | 14 ++++---- python_utils/formatters.py | 9 ++--- python_utils/generators.py | 28 ++++++++++----- python_utils/logger.py | 6 ++-- python_utils/loguru.py | 5 +-- python_utils/terminal.py | 39 ++++++--------------- python_utils/time.py | 71 +++++++++++++++++++++++++++++--------- python_utils/types.py | 19 +++------- setup.py | 2 +- 15 files changed, 212 insertions(+), 164 deletions(-) delete mode 100644 pyrightconfig.json diff --git a/.gitignore b/.gitignore index a04bcd0..46105bf 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ /docs/_build /cover /.eggs +/.* \ No newline at end of file diff --git a/pyrightconfig.json b/pyrightconfig.json deleted file mode 100644 index aaf3faf..0000000 --- a/pyrightconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "include": [ - "python_utils" - ], - "exclude": [ - "python_utils/types.py", - ], -} diff --git a/python_utils/aio.py b/python_utils/aio.py index de1ff87..ef17cb6 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -7,17 +7,19 @@ from . import types +_N = types.TypeVar('_N', int, float) + async def acount( - start: float = 0, - step: float = 1, + start: _N = 0, + step: _N = 1, delay: float = 0, - stop: types.Optional[float] = None, -) -> types.AsyncIterator[float]: + stop: types.Optional[_N] = None, +) -> types.AsyncIterator[_N]: '''Asyncio version of itertools.count()''' for item in itertools.count(start, step): # pragma: no branch if stop is not None and item >= stop: break - yield item + yield types.cast(_N, item) await asyncio.sleep(delay) diff --git a/python_utils/containers.py b/python_utils/containers.py index f8920a8..187b816 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,3 +1,4 @@ +# pyright: reportIncompatibleMethodOverride=false import abc import typing from typing import Any, Generator @@ -22,28 +23,32 @@ # Using types.Union instead of | since Python 3.7 doesn't fully support it DictUpdateArgs = types.Union[ - types.Mapping, - types.Iterable[types.Union[types.Tuple[Any, Any], types.Mapping]], + types.Mapping[types.Any, types.Any], + types.Iterable[ + types.Union[types.Tuple[Any, Any], types.Mapping[types.Any, types.Any]] + ], '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] class CastedDictBase(types.Dict[KT, VT], abc.ABC): - _key_cast: KT_cast - _value_cast: VT_cast + _key_cast: KT_cast[KT] + _value_cast: VT_cast[VT] def __init__( self, - key_cast: KT_cast = None, - value_cast: VT_cast = None, - *args: DictUpdateArgs, + key_cast: KT_cast[KT] = None, + value_cast: VT_cast[VT] = None, + *args: DictUpdateArgs[KT, VT], **kwargs: VT, ) -> None: self._value_cast = value_cast self._key_cast = key_cast self.update(*args, **kwargs) - def update(self, *args: DictUpdateArgs, **kwargs: VT) -> None: + def update( + self, *args: DictUpdateArgs[types.Any, types.Any], **kwargs: types.Any + ) -> None: if args: kwargs.update(*args) @@ -93,7 +98,7 @@ class CastedDict(CastedDictBase[KT, VT]): {1: 2, '3': '4', '5': '6', '7': '8'} ''' - def __setitem__(self, key: Any, value: Any) -> None: + def __setitem__(self, key: typing.Any, value: typing.Any) -> None: if self._value_cast is not None: value = self._value_cast(value) @@ -146,13 +151,13 @@ class LazyCastedDict(CastedDictBase[KT, VT]): '4' ''' - def __setitem__(self, key: Any, value: Any) -> None: + def __setitem__(self, key: types.Any, value: types.Any): if self._key_cast is not None: key = self._key_cast(key) super().__setitem__(key, value) - def __getitem__(self, key: Any) -> VT: + def __getitem__(self, key: types.Any) -> VT: if self._key_cast is not None: key = self._key_cast(key) @@ -163,9 +168,7 @@ def __getitem__(self, key: Any) -> VT: return value - def items( # type: ignore - self, - ) -> Generator[types.Tuple[KT, VT], None, None]: + def items(self) -> Generator[tuple[KT, VT], None, None]: # type: ignore if self._value_cast is None: yield from super().items() else: @@ -247,7 +250,7 @@ def append(self, value: HT) -> None: self._set.add(value) super().append(value) - def __contains__(self, item): + def __contains__(self, item: HT) -> bool: # type: ignore return item in self._set @types.overload @@ -258,29 +261,37 @@ def __setitem__(self, indices: types.SupportsIndex, values: HT) -> None: def __setitem__(self, indices: slice, values: types.Iterable[HT]) -> None: ... - def __setitem__(self, indices, values) -> None: + def __setitem__( + self, + indices: types.Union[slice, types.SupportsIndex], + values: types.Union[types.Iterable[HT], HT], + ) -> None: if isinstance(indices, slice): + values = types.cast(types.Iterable[HT], values) if self.on_duplicate == 'ignore': raise RuntimeError( 'ignore mode while setting slices introduces ambiguous ' 'behaviour and is therefore not supported' ) - duplicates = set(values) & self._set - if duplicates and values != self[indices]: - raise ValueError('Duplicate values: %s' % duplicates) + duplicates: types.Set[HT] = set(values) & self._set + if duplicates and values != list(self[indices]): + raise ValueError(f'Duplicate values: {duplicates}') self._set.update(values) - super().__setitem__(indices, values) else: + values = types.cast(HT, values) if values in self._set and values != self[indices]: if self.on_duplicate == 'raise': - raise ValueError('Duplicate value: %s' % values) + raise ValueError(f'Duplicate value: {values}') else: return self._set.add(values) - super().__setitem__(indices, values) + + super().__setitem__( + types.cast(slice, indices), types.cast(types.List[HT], values) + ) def __delitem__( self, index: types.Union[types.SupportsIndex, slice] diff --git a/python_utils/converters.py b/python_utils/converters.py index c1eba73..e8c91f6 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -5,12 +5,14 @@ from . import types +_TN = types.TypeVar('_TN', bound=types.DecimalNumber) + def to_int( input_: typing.Optional[str] = None, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.Optional[types.Pattern] = None, + regexp: types.Optional[types.Pattern[str]] = None, ) -> int: r''' Convert the given input to an integer or return default @@ -84,8 +86,7 @@ def to_int( try: if regexp and input_: - match = regexp.search(input_) - if match: + if match := regexp.search(input_): input_ = match.groups()[-1] if input_ is None: @@ -100,7 +101,7 @@ def to_float( input_: str, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.Optional[types.Pattern] = None, + regexp: types.Optional[types.Pattern[str]] = None, ) -> types.Number: r''' Convert the given `input_` to an integer or return default @@ -165,8 +166,7 @@ def to_float( try: if regexp: - match = regexp.search(input_) - if match: + if match := regexp.search(input_): input_ = match.group(1) return float(input_) except exception: @@ -223,9 +223,7 @@ def to_str( >>> to_str(Foo) "" ''' - if isinstance(input_, bytes): - pass - else: + if not isinstance(input_, bytes): if not hasattr(input_, 'encode'): input_ = str(input_) @@ -263,12 +261,12 @@ def scale_1024( def remap( - value: types.DecimalNumber, - old_min: types.DecimalNumber, - old_max: types.DecimalNumber, - new_min: types.DecimalNumber, - new_max: types.DecimalNumber, -) -> types.DecimalNumber: + value: _TN, + old_min: _TN, + old_max: _TN, + new_min: _TN, + new_max: _TN, +) -> _TN: ''' remap a value from one range into another. @@ -362,24 +360,22 @@ def remap( else: type_ = int - value = type_(value) - old_min = type_(old_min) - old_max = type_(old_max) - new_max = type_(new_max) - new_min = type_(new_min) + value = types.cast(_TN, type_(value)) + old_min = types.cast(_TN, type_(old_min)) + old_max = types.cast(_TN, type_(old_max)) + new_max = types.cast(_TN, type_(new_max)) + new_min = types.cast(_TN, type_(new_min)) - old_range = old_max - old_min # type: ignore - new_range = new_max - new_min # type: ignore + # These might not be floats but the Python type system doesn't understand the + # generic type system in this case + old_range = types.cast(float, old_max) - types.cast(float, old_min) + new_range = types.cast(float, new_max) - types.cast(float, new_min) if old_range == 0: - raise ValueError( - 'Input range ({}-{}) is empty'.format(old_min, old_max) - ) + raise ValueError(f'Input range ({old_min}-{old_max}) is empty') if new_range == 0: - raise ValueError( - 'Output range ({}-{}) is empty'.format(new_min, new_max) - ) + raise ValueError(f'Output range ({new_min}-{new_max}) is empty') new_value = (value - old_min) * new_range # type: ignore @@ -390,4 +386,4 @@ def remap( new_value += new_min # type: ignore - return new_value + return types.cast(_TN, new_value) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 6c237aa..a5484e2 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -3,8 +3,12 @@ import random from . import types +T = types.TypeVar('T') +TC = types.TypeVar('TC', bound=types.Container[types.Any]) +P = types.ParamSpec('P') -def set_attributes(**kwargs): + +def set_attributes(**kwargs: types.Any) -> types.Callable[..., types.Any]: '''Decorator to set attributes on functions and classes A common usage for this pattern is the Django Admin where @@ -28,7 +32,9 @@ def set_attributes(**kwargs): ''' - def _set_attributes(function): + def _set_attributes( + function: types.Callable[P, T] + ) -> types.Callable[P, T]: for key, value in kwargs.items(): setattr(function, key, value) return function @@ -36,7 +42,13 @@ def _set_attributes(function): return _set_attributes -def listify(collection: types.Callable = list, allow_empty: bool = True): +def listify( + collection: types.Callable[[types.Iterable[T]], TC] = list, # type: ignore + allow_empty: bool = True, +) -> types.Callable[ + [types.Callable[..., types.Optional[types.Iterable[T]]]], + types.Callable[..., TC], +]: ''' Convert any generator to a list or other type of collection. @@ -60,10 +72,10 @@ def listify(collection: types.Callable = list, allow_empty: bool = True): ... def empty_generator_not_allowed(): ... pass - >>> empty_generator_not_allowed() + >>> empty_generator_not_allowed() # doctest: +ELLIPSIS Traceback (most recent call last): ... - TypeError: 'NoneType' object is not iterable + TypeError: ... `allow_empty` is `False` >>> @listify(collection=set) ... def set_generator(): @@ -83,13 +95,22 @@ def listify(collection: types.Callable = list, allow_empty: bool = True): {'a': 1, 'b': 2} ''' - def _listify(function): - @functools.wraps(function) - def __listify(*args, **kwargs): - result = function(*args, **kwargs) - if result is None and allow_empty: - return [] - return collection(result) + def _listify( + function: types.Callable[..., types.Optional[types.Iterable[T]]] + ) -> types.Callable[..., TC]: + def __listify(*args: types.Any, **kwargs: types.Any) -> TC: + result: types.Optional[types.Iterable[T]] = function( + *args, **kwargs + ) + if result is None: + if allow_empty: + return collection(iter(())) + else: + raise TypeError( + f'{function} returned `None` and `allow_empty` is `False`' + ) + else: + return collection(result) return __listify @@ -109,12 +130,13 @@ def sample(sample_rate: float): ... return 1 Calls to *demo_function* will be limited to 50% approximatly. - ''' - def _sample(function): + def _sample( + function: types.Callable[P, T] + ) -> types.Callable[P, types.Optional[T]]: @functools.wraps(function) - def __sample(*args, **kwargs): + def __sample(*args: P.args, **kwargs: P.kwargs) -> types.Optional[T]: if random.random() < sample_rate: return function(*args, **kwargs) else: @@ -124,6 +146,7 @@ def __sample(*args, **kwargs): args, kwargs, ) # noqa: E501 + return None return __sample diff --git a/python_utils/exceptions.py b/python_utils/exceptions.py index 14855c5..3ffc01a 100644 --- a/python_utils/exceptions.py +++ b/python_utils/exceptions.py @@ -1,11 +1,11 @@ -import typing +from . import types def raise_exception( - exception_class: typing.Type[Exception], - *args: typing.Any, - **kwargs: typing.Any, -) -> typing.Callable: + exception_class: types.Type[Exception], + *args: types.Any, + **kwargs: types.Any, +) -> types.Callable[..., None]: ''' Returns a function that raises an exception of the given type with the given arguments. @@ -16,11 +16,11 @@ def raise_exception( ValueError: spam ''' - def raise_(*args_: typing.Any, **kwargs_: typing.Any) -> typing.Any: + def raise_(*args_: types.Any, **kwargs_: types.Any) -> types.Any: raise exception_class(*args, **kwargs) return raise_ -def reraise(*args: typing.Any, **kwargs: typing.Any) -> typing.Any: +def reraise(*args: types.Any, **kwargs: types.Any) -> types.Any: raise diff --git a/python_utils/formatters.py b/python_utils/formatters.py index d2d76cb..04d661d 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -1,3 +1,4 @@ +# pyright: reportUnnecessaryIsInstance=false import datetime from python_utils import types @@ -21,7 +22,7 @@ def camel_to_underscore(name: str) -> str: >>> camel_to_underscore('__SpamANDBacon__') '__spam_and_bacon__' ''' - output = [] + output: types.List[str] = [] for i, c in enumerate(name): if i > 0: pc = name[i - 1] @@ -44,7 +45,7 @@ def camel_to_underscore(name: str) -> str: def apply_recursive( function: types.Callable[[str], str], data: types.OptionalScope = None, - **kwargs, + **kwargs: types.Any, ) -> types.OptionalScope: ''' Apply a function to all keys in a scope recursively @@ -137,7 +138,7 @@ def timesince( (diff.seconds % 60, 'second', 'seconds'), ) - output = [] + output: types.List[str] = [] for period, singular, plural in periods: if int(period): if int(period) == 1: @@ -146,6 +147,6 @@ def timesince( output.append('%d %s' % (period, plural)) if output: - return '%s ago' % ' and '.join(output[:2]) + return f'{" and ".join(output[:2])} ago' return default diff --git a/python_utils/generators.py b/python_utils/generators.py index 2930334..b827ed4 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -5,16 +5,19 @@ from python_utils import types +_T = types.TypeVar('_T') + + async def abatcher( - generator: types.AsyncIterator, + generator: types.AsyncGenerator[_T, None], batch_size: types.Optional[int] = None, interval: types.Optional[types.delta_type] = None, -) -> types.AsyncIterator[list]: +) -> types.AsyncGenerator[types.List[_T], None]: ''' Asyncio generator wrapper that returns items with a given batch size or interval (whichever is reached first). ''' - batch: list = [] + batch: types.List[_T] = [] assert batch_size or interval, 'Must specify either batch_size or interval' @@ -26,16 +29,22 @@ async def abatcher( # Set the timeout to 10 years interval_s = 60 * 60 * 24 * 365 * 10.0 - next_yield = time.perf_counter() + interval_s + next_yield: float = time.perf_counter() + interval_s - pending: types.Set = set() + done: types.Set[asyncio.Task[_T]] + pending: types.Set[asyncio.Task[_T]] = set() while True: try: done, pending = await asyncio.wait( pending or [ - asyncio.create_task(generator.__anext__()), # type: ignore + asyncio.create_task( + types.cast( + types.Coroutine[None, None, _T], + generator.__anext__(), + ) + ), ], timeout=interval_s, return_when=asyncio.FIRST_COMPLETED, @@ -65,12 +74,13 @@ async def abatcher( def batcher( - iterable: types.Iterable, batch_size: int = 10 -) -> types.Iterator[list]: + iterable: types.Iterable[_T], + batch_size: int = 10, +) -> types.Generator[types.List[_T], None, None]: ''' Generator wrapper that returns items with a given batch size ''' - batch = [] + batch: types.List[_T] = [] for item in iterable: batch.append(item) if len(batch) == batch_size: diff --git a/python_utils/logger.py b/python_utils/logger.py index 10ee5e6..bcf0bab 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -33,7 +33,9 @@ class LoggerBase(abc.ABC): logger: typing.Any @classmethod - def __get_name(cls, *name_parts: str) -> str: + def __get_name( # pyright: ignore[reportUnusedFunction] + cls, *name_parts: str + ) -> str: return '.'.join(n.strip() for n in name_parts if n.strip()) @classmethod @@ -95,7 +97,7 @@ class Logged(LoggerBase): def __get_name(cls, *name_parts: str) -> str: return LoggerBase._LoggerBase__get_name(*name_parts) # type: ignore - def __new__(cls, *args, **kwargs): + def __new__(cls, *args: typing.Any, **kwargs: typing.Any): cls.logger = logging.getLogger( cls.__get_name(cls.__module__, cls.__name__) ) diff --git a/python_utils/loguru.py b/python_utils/loguru.py index c09fd4f..22b258d 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -1,8 +1,9 @@ from __future__ import annotations +import typing import loguru -from . import logger as logger_module, types +from . import logger as logger_module __all__ = ['Logurud'] @@ -10,6 +11,6 @@ class Logurud(logger_module.LoggerBase): logger: loguru.Logger - def __new__(cls, *args: types.Any, **kwargs: types.Any) -> Logurud: + def __new__(cls, *args: typing.Any, **kwargs: typing.Any): cls.logger: loguru.Logger = loguru.logger.opt(depth=1) return super().__new__(cls) diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 19c494d..53948d8 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -1,3 +1,4 @@ +import contextlib import os import typing @@ -20,7 +21,7 @@ def get_terminal_size() -> Dimensions: # pragma: no cover w: typing.Optional[int] h: typing.Optional[int] - try: + with contextlib.suppress(Exception): # Default to 79 characters for IPython notebooks from IPython import get_ipython # type: ignore @@ -29,10 +30,7 @@ def get_terminal_size() -> Dimensions: # pragma: no cover if isinstance(ipython, zmqshell.ZMQInteractiveShell): return 79, 24 - except Exception: # pragma: no cover - pass - - try: + with contextlib.suppress(Exception): # This works for Python 3, but not Pypy3. Probably the best method if # it's supported so let's always try import shutil @@ -42,18 +40,12 @@ def get_terminal_size() -> Dimensions: # pragma: no cover # The off by one is needed due to progressbars in some cases, for # safety we'll always substract it. return w - 1, h - except Exception: # pragma: no cover - pass - - try: + with contextlib.suppress(Exception): w = converters.to_int(os.environ.get('COLUMNS')) h = converters.to_int(os.environ.get('LINES')) if w and h: return w, h - except Exception: # pragma: no cover - pass - - try: + with contextlib.suppress(Exception): import blessings # type: ignore terminal = blessings.Terminal() @@ -61,32 +53,23 @@ def get_terminal_size() -> Dimensions: # pragma: no cover h = terminal.height if w and h: return w, h - except Exception: # pragma: no cover - pass - - try: + with contextlib.suppress(Exception): # The method can return None so we don't unpack it wh = _get_terminal_size_linux() if wh is not None and all(wh): return wh - except Exception: # pragma: no cover - pass - try: + with contextlib.suppress(Exception): # Windows detection doesn't always work, let's try anyhow wh = _get_terminal_size_windows() if wh is not None and all(wh): return wh - except Exception: # pragma: no cover - pass - try: + with contextlib.suppress(Exception): # needed for window's python in cygwin's xterm! wh = _get_terminal_size_tput() if wh is not None and all(wh): return wh - except Exception: # pragma: no cover - pass return 79, 24 @@ -162,12 +145,10 @@ def ioctl_GWINSZ(fd): size = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) if not size: - try: - fd = os.open(os.ctermid(), os.O_RDONLY) + with contextlib.suppress(Exception): + fd = os.open(os.ctermid(), os.O_RDONLY) # type: ignore size = ioctl_GWINSZ(fd) os.close(fd) - except Exception: - pass if not size: try: size = os.environ['LINES'], os.environ['COLUMNS'] diff --git a/python_utils/time.py b/python_utils/time.py index ce1d91c..4084a5a 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -1,3 +1,4 @@ +# pyright: reportUnnecessaryIsInstance=false import asyncio import datetime import functools @@ -7,6 +8,10 @@ import python_utils from python_utils import aio, exceptions, types +_T = types.TypeVar('_T') +_P = types.ParamSpec('_P') + + # There might be a better way to get the epoch with tzinfo, please create # a pull request if you know a better way that functions for Python 2 and 3 epoch = datetime.datetime(year=1970, month=1, day=1) @@ -139,7 +144,9 @@ def format_time( def timeout_generator( timeout: types.delta_type, interval: types.delta_type = datetime.timedelta(seconds=1), - iterable: types.Union[types.Iterable, types.Callable] = itertools.count, + iterable: types.Union[ + types.Iterable[_T], types.Callable[[], types.Iterable[_T]] + ] = itertools.count, # type: ignore interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, ): @@ -179,7 +186,7 @@ def timeout_generator( maximum_interval ) - iterable_: types.Iterable + iterable_: types.Iterable[_T] if callable(iterable): iterable_ = iterable() else: @@ -202,12 +209,14 @@ def timeout_generator( async def aio_timeout_generator( timeout: types.delta_type, interval: types.delta_type = datetime.timedelta(seconds=1), - iterable: types.Union[types.AsyncIterable, types.Callable] = aio.acount, + iterable: types.Union[ + types.AsyncIterable[_T], types.Callable[..., types.AsyncIterable[_T]] + ] = aio.acount, interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, -): +) -> types.AsyncGenerator[_T, None]: ''' - Aync generator that walks through the given iterable (a counter by + Async generator that walks through the given async iterable (a counter by default) until the float_timeout is reached with a configurable float_interval between items @@ -226,7 +235,7 @@ async def aio_timeout_generator( maximum_interval ) - iterable_: types.AsyncIterable + iterable_: types.AsyncIterable[_T] if callable(iterable): iterable_ = iterable() else: @@ -247,12 +256,22 @@ async def aio_timeout_generator( async def aio_generator_timeout_detector( - generator: types.AsyncGenerator, + generator: types.AsyncGenerator[_T, None], timeout: types.Optional[types.delta_type] = None, total_timeout: types.Optional[types.delta_type] = None, - on_timeout: types.Optional[types.Callable] = exceptions.reraise, - **kwargs, -): + on_timeout: types.Optional[ + types.Callable[ + [ + types.AsyncGenerator[_T, None], + types.Optional[types.delta_type], + types.Optional[types.delta_type], + BaseException, + ], + types.Any, + ] + ] = exceptions.reraise, + **on_timeout_kwargs: types.Mapping[types.Text, types.Any], +) -> types.AsyncGenerator[_T, None]: ''' This function is used to detect if an asyncio generator has not yielded an element for a set amount of time. @@ -286,7 +305,11 @@ async def aio_generator_timeout_detector( except asyncio.TimeoutError as exception: if on_timeout is not None: await on_timeout( - generator, timeout, total_timeout, exception, **kwargs + generator, + timeout, + total_timeout, + exception, + **on_timeout_kwargs, ) break @@ -297,26 +320,40 @@ async def aio_generator_timeout_detector( def aio_generator_timeout_detector_decorator( timeout: types.Optional[types.delta_type] = None, total_timeout: types.Optional[types.delta_type] = None, - on_timeout: types.Optional[types.Callable] = exceptions.reraise, - **kwargs, + on_timeout: types.Optional[ + types.Callable[ + [ + types.AsyncGenerator[types.Any, None], + types.Optional[types.delta_type], + types.Optional[types.delta_type], + BaseException, + ], + types.Any, + ] + ] = exceptions.reraise, + **on_timeout_kwargs: types.Mapping[types.Text, types.Any], ): ''' A decorator wrapper for aio_generator_timeout_detector. ''' - def _timeout_detector_decorator(generator: types.Callable): + def _timeout_detector_decorator( + generator: types.Callable[_P, types.AsyncGenerator[_T, None]] + ) -> types.Callable[_P, types.AsyncGenerator[_T, None]]: ''' The decorator itself. ''' @functools.wraps(generator) - def wrapper(*args, **wrapper_kwargs): + def wrapper( + *args: _P.args, **kwargs: _P.kwargs + ) -> types.AsyncGenerator[_T, None]: return aio_generator_timeout_detector( - generator(*args, **wrapper_kwargs), + generator(*args, **kwargs), timeout, total_timeout, on_timeout, - **kwargs, + **on_timeout_kwargs, ) return wrapper diff --git a/python_utils/types.py b/python_utils/types.py index 1e749c8..1dfcd63 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,15 +1,11 @@ +# pyright: reportWildcardImportFromLibrary=false import datetime import decimal -import sys -from typing import * # type: ignore # pragma: no cover +from typing_extensions import * # type: ignore # pragma: no cover # noqa: F403 +from typing import * # type: ignore # pragma: no cover # noqa: F403 -# import * does not import Pattern -from typing import Pattern - -if sys.version_info >= (3, 8): # pragma: no cover - from typing import Literal, SupportsIndex -else: # pragma: no cover - from typing_extensions import Literal, SupportsIndex +# import * does not import these in all Python versions +from typing import Pattern, BinaryIO, IO, TextIO, Match # Quickhand for optional because it gets so much use. If only Python had # support for an optional type shorthand such as `SomeType?` instead of @@ -39,10 +35,6 @@ None, ] -assert Pattern is not None # type: ignore -assert Literal is not None -assert SupportsIndex is not None - __all__ = [ 'OptionalScope', 'Number', @@ -141,5 +133,4 @@ 'TYPE_CHECKING', 'TypeAlias', 'TypeGuard', - 'TracebackType', ] diff --git a/setup.py b/setup.py index af90617..b5ae708 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ ), package_data={'python_utils': ['py.typed']}, long_description=long_description, - install_requires=['typing_extensions;python_version<"3.8"'], + install_requires=['typing_extensions'], tests_require=['pytest'], extras_require={ 'loguru': [ From f96fbe0b2b3b52b7eac91c6e02d0ed210a681750 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 26 May 2023 13:02:07 +0200 Subject: [PATCH 083/132] Added AIO tests --- _python_utils_tests/test_aio.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 _python_utils_tests/test_aio.py diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py new file mode 100644 index 0000000..3e275cd --- /dev/null +++ b/_python_utils_tests/test_aio.py @@ -0,0 +1,20 @@ +from datetime import datetime +import pytest +import asyncio +from python_utils.aio import acount + + +@pytest.mark.asyncio +async def test_acount(monkeypatch: pytest.MonkeyPatch): + sleeps: list[float] = [] + + async def mock_sleep(delay: float): + sleeps.append(delay) + + monkeypatch.setattr(asyncio, 'sleep', mock_sleep) + + async for i in acount(delay=1, stop=3.5): + print('i', i, datetime.now()) + + assert len(sleeps) == 4 + assert sum(sleeps) == 4 From 4f14340ae50f974dfe049c9e4c24333ffe055a83 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 26 May 2023 13:02:37 +0200 Subject: [PATCH 084/132] Added pyproject.toml for black and pyright configuration --- pyproject.toml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2720bf2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,11 @@ +[tool.black] +line-length = 79 +target-version = ['py37', 'py38', 'py39', 'py310', 'py311'] +skip-string-normalization = true + +[tool.pyright] +include = ['python_utils'] +strict = ['python_utils', '_python_utils_tests/test_aio.py'] +# The terminal file is very OS specific and dependent on imports so we're skipping it from type checking +ignore = ['python_utils/terminal.py'] +pythonVersion = '3.8' \ No newline at end of file From c7a3c467a285b41aae5465535563ce859480d66f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 26 May 2023 13:28:20 +0200 Subject: [PATCH 085/132] Added types from the `types` module --- python_utils/types.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/python_utils/types.py b/python_utils/types.py index 1dfcd63..cd63b53 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -3,6 +3,7 @@ import decimal from typing_extensions import * # type: ignore # pragma: no cover # noqa: F403 from typing import * # type: ignore # pragma: no cover # noqa: F403 +from types import * # type: ignore # pragma: no cover # noqa: F403 # import * does not import these in all Python versions from typing import Pattern, BinaryIO, IO, TextIO, Match @@ -55,13 +56,15 @@ 'SupportsIndex', 'Optional', 'ParamSpec', + 'ParamSpecArgs', + 'ParamSpecKwargs', 'Protocol', 'Tuple', 'Type', 'TypeVar', 'Union', # ABCs (from collections.abc). - 'AbstractSet', # collections.abc.Set. + 'AbstractSet', 'ByteString', 'Container', 'ContextManager', @@ -126,11 +129,36 @@ 'no_type_check_decorator', 'NoReturn', 'overload', - 'ParamSpecArgs', - 'ParamSpecKwargs', 'runtime_checkable', 'Text', 'TYPE_CHECKING', 'TypeAlias', 'TypeGuard', + 'TracebackType', + # Types from the `types` module. + 'FunctionType', + 'LambdaType', + 'CodeType', + 'MappingProxyType', + 'SimpleNamespace', + 'GeneratorType', + 'CoroutineType', + 'AsyncGeneratorType', + 'MethodType', + 'BuiltinFunctionType', + 'BuiltinMethodType', + 'WrapperDescriptorType', + 'MethodWrapperType', + 'MethodDescriptorType', + 'ClassMethodDescriptorType', + 'ModuleType', + 'TracebackType', + 'FrameType', + 'GetSetDescriptorType', + 'MemberDescriptorType', + 'new_class', + 'resolve_bases', + 'prepare_class', + 'DynamicClassAttribute', + 'coroutine', ] From e43a51279ccca32c001933d6915903578595808b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 26 May 2023 13:28:34 +0200 Subject: [PATCH 086/132] Fixed tests and more Python version compatibility --- _python_utils_tests/test_aio.py | 4 +++- pytest.ini | 2 +- python_utils/containers.py | 20 ++++++++++---------- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index 3e275cd..0eb95d3 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -1,12 +1,14 @@ from datetime import datetime import pytest import asyncio + +from python_utils import types from python_utils.aio import acount @pytest.mark.asyncio async def test_acount(monkeypatch: pytest.MonkeyPatch): - sleeps: list[float] = [] + sleeps: types.List[float] = [] async def mock_sleep(delay: float): sleeps.append(delay) diff --git a/pytest.ini b/pytest.ini index 5d49701..a8e632a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,7 +7,7 @@ addopts = --doctest-modules --cov python_utils --cov-report term-missing - --mypy +; --mypy doctest_optionflags = ALLOW_UNICODE diff --git a/python_utils/containers.py b/python_utils/containers.py index 187b816..31700e5 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,7 +1,6 @@ # pyright: reportIncompatibleMethodOverride=false import abc import typing -from typing import Any, Generator from . import types @@ -15,18 +14,17 @@ #: A type alias for a dictionary with keys of type KT and values of type VT. DT = types.Dict[KT, VT] #: A type alias for the casted type of a dictionary key. -KT_cast = types.Optional[types.Callable[[Any], KT]] +KT_cast = types.Optional[types.Callable[..., KT]] #: A type alias for the casted type of a dictionary value. -VT_cast = types.Optional[types.Callable[[Any], VT]] +VT_cast = types.Optional[types.Callable[..., VT]] #: A type alias for the hashable values of the `UniqueList` HT = types.TypeVar('HT', bound=types.Hashable) # Using types.Union instead of | since Python 3.7 doesn't fully support it DictUpdateArgs = types.Union[ - types.Mapping[types.Any, types.Any], - types.Iterable[ - types.Union[types.Tuple[Any, Any], types.Mapping[types.Any, types.Any]] - ], + types.Mapping[KT, VT], + types.Iterable[types.Tuple[KT, VT]], + types.Iterable[types.Mapping[KT, VT]], '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] @@ -56,7 +54,7 @@ def update( for key, value in kwargs.items(): self[key] = value - def __setitem__(self, key: Any, value: Any) -> None: + def __setitem__(self, key: types.Any, value: types.Any) -> None: if self._key_cast is not None: key = self._key_cast(key) @@ -168,14 +166,16 @@ def __getitem__(self, key: types.Any) -> VT: return value - def items(self) -> Generator[tuple[KT, VT], None, None]: # type: ignore + def items( # type: ignore + self, + ) -> types.Generator[types.Tuple[KT, VT], None, None]: if self._value_cast is None: yield from super().items() else: for key, value in super().items(): yield key, self._value_cast(value) - def values(self) -> Generator[VT, None, None]: # type: ignore + def values(self) -> types.Generator[VT, None, None]: # type: ignore if self._value_cast is None: yield from super().values() else: From 0592a5fbb62aa7e410f487fc631d2a98ee8c74e1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 25 May 2023 18:15:46 +0200 Subject: [PATCH 087/132] Added sliceable deque --- python_utils/containers.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/python_utils/containers.py b/python_utils/containers.py index 31700e5..53dcd57 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -305,6 +305,38 @@ def __delitem__( super().__delitem__(index) +class SlicableDeque(Generic[T], deque): + def __getitem__(self, index: Union[int, slice]) -> Union[T, 'SlicableDeque[T]']: + """ + Return the item or slice at the given index. + + >>> d = SlicableDeque[int]([1, 2, 3, 4, 5]) + >>> d[1:4] + SlicableDeque([2, 3, 4]) + + >>> d = SlicableDeque[str](['a', 'b', 'c']) + >>> d[-2:] + SlicableDeque(['b', 'c']) + + """ + if isinstance(index, slice): + start, stop, step = index.indices(len(self)) + return self.__class__(self[i] for i in range(start, stop, step)) + else: + return super().__getitem__(index) + + def pop(self) -> T: + """ + Remove and return the rightmost element. + + >>> d = SlicableDeque[float]([1.5, 2.5, 3.5]) + >>> d.pop() + 3.5 + + """ + return super().pop() + + if __name__ == '__main__': import doctest From 5eac0d3297187fea1194a04da5e26c1dabe60052 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 29 May 2023 02:02:04 +0200 Subject: [PATCH 088/132] Added type hinting to new container classes --- python_utils/containers.py | 38 +++++++++++++++++++++----------------- python_utils/converters.py | 4 ++-- python_utils/decorators.py | 3 ++- python_utils/types.py | 2 +- tox.ini | 4 +--- 5 files changed, 27 insertions(+), 24 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index 53dcd57..4685ab9 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,6 +1,7 @@ # pyright: reportIncompatibleMethodOverride=false import abc import typing +import collections from . import types @@ -19,6 +20,8 @@ VT_cast = types.Optional[types.Callable[..., VT]] #: A type alias for the hashable values of the `UniqueList` HT = types.TypeVar('HT', bound=types.Hashable) +#: A type alias for a regular generic type +T = types.TypeVar('T') # Using types.Union instead of | since Python 3.7 doesn't fully support it DictUpdateArgs = types.Union[ @@ -28,6 +31,8 @@ '_typeshed.SupportsKeysAndGetItem[KT, VT]', ] +OnDuplicate = types.Literal['ignore', 'raise'] + class CastedDictBase(types.Dict[KT, VT], abc.ABC): _key_cast: KT_cast[KT] @@ -222,7 +227,7 @@ class UniqueList(types.List[HT]): def __init__( self, *args: HT, - on_duplicate: types.Literal['raise', 'ignore'] = 'ignore', + on_duplicate: OnDuplicate = 'ignore', ): self.on_duplicate = on_duplicate self._set = set() @@ -305,9 +310,19 @@ def __delitem__( super().__delitem__(index) -class SlicableDeque(Generic[T], deque): - def __getitem__(self, index: Union[int, slice]) -> Union[T, 'SlicableDeque[T]']: - """ +class SlicableDeque(types.Generic[T], collections.deque): # type: ignore + @types.overload + def __getitem__(self, index: types.SupportsIndex) -> T: + ... + + @types.overload + def __getitem__(self, index: slice) -> 'SlicableDeque[T]': + ... + + def __getitem__( + self, index: types.Union[types.SupportsIndex, slice] + ) -> types.Union[T, 'SlicableDeque[T]']: + ''' Return the item or slice at the given index. >>> d = SlicableDeque[int]([1, 2, 3, 4, 5]) @@ -318,23 +333,12 @@ def __getitem__(self, index: Union[int, slice]) -> Union[T, 'SlicableDeque[T]']: >>> d[-2:] SlicableDeque(['b', 'c']) - """ + ''' if isinstance(index, slice): start, stop, step = index.indices(len(self)) return self.__class__(self[i] for i in range(start, stop, step)) else: - return super().__getitem__(index) - - def pop(self) -> T: - """ - Remove and return the rightmost element. - - >>> d = SlicableDeque[float]([1.5, 2.5, 3.5]) - >>> d.pop() - 3.5 - - """ - return super().pop() + return types.cast(T, super().__getitem__(index)) if __name__ == '__main__': diff --git a/python_utils/converters.py b/python_utils/converters.py index e8c91f6..68438ee 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -366,8 +366,8 @@ def remap( new_max = types.cast(_TN, type_(new_max)) new_min = types.cast(_TN, type_(new_min)) - # These might not be floats but the Python type system doesn't understand the - # generic type system in this case + # These might not be floats but the Python type system doesn't understand + # the generic type system in this case old_range = types.cast(float, old_max) - types.cast(float, old_min) new_range = types.cast(float, new_max) - types.cast(float, new_min) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index a5484e2..6559cf0 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -107,7 +107,8 @@ def __listify(*args: types.Any, **kwargs: types.Any) -> TC: return collection(iter(())) else: raise TypeError( - f'{function} returned `None` and `allow_empty` is `False`' + f'{function} returned `None` and `allow_empty` ' + 'is `False`' ) else: return collection(result) diff --git a/python_utils/types.py b/python_utils/types.py index cd63b53..01c319a 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,7 +1,7 @@ # pyright: reportWildcardImportFromLibrary=false import datetime import decimal -from typing_extensions import * # type: ignore # pragma: no cover # noqa: F403 +from typing_extensions import * # type: ignore # noqa: F403 from typing import * # type: ignore # pragma: no cover # noqa: F403 from types import * # type: ignore # pragma: no cover # noqa: F403 diff --git a/tox.ini b/tox.ini index 8290500..f017232 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,13 @@ [tox] -envlist = black, py37, py38, py39, py310, py311, pypy3, flake8, docs, mypy, pyright +envlist = black, py38, py39, py310, py311, flake8, docs, mypy, pyright skip_missing_interpreters = True [testenv] basepython = - py37: python3.7 py38: python3.8 py39: python3.9 py310: python3.10 py311: python3.11 - pypy: pypy setenv = PY_IGNORE_IMPORTMISMATCH=1 deps = From 95b5d3a5e84074e46345e5b1f414e446d9e28ac0 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 29 May 2023 02:11:31 +0200 Subject: [PATCH 089/132] dropped python 3.7 from tests --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b7d08dc..2f8ac44 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 4 strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11-dev'] + python-version: ['3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v3 From f50712a2856ae123f68b697eacbaa2d140813d9a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 29 May 2023 02:57:31 +0200 Subject: [PATCH 090/132] Dropped Python 3.7 support --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b5ae708..4b14c0b 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ if __name__ == '__main__': setuptools.setup( - python_requires='>3.6.0', + python_requires='>3.8.0', name='python-utils', version=about['__version__'], author=about['__author__'], From e68d6e4fdfeb9fe757fd8cb73c88fe745aa7003f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 29 May 2023 02:58:48 +0200 Subject: [PATCH 091/132] Incrementing version to v3.6.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 7e57bd8..69bddae 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.5.2' +__version__ = '3.6.0' From 83f04bd7e3b0bdfc2059bfe67bad58c3a31b78b3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 17 Jun 2023 13:33:12 +0200 Subject: [PATCH 092/132] Increased `typing-extensions` version requirement to fix #38 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4b14c0b..306ace7 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ ), package_data={'python_utils': ['py.typed']}, long_description=long_description, - install_requires=['typing_extensions'], + install_requires=['typing_extensions>3.10.0.2'], tests_require=['pytest'], extras_require={ 'loguru': [ From 7e1b6206cb52288752989fcb6701d18c668e0489 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 17 Jun 2023 13:33:41 +0200 Subject: [PATCH 093/132] Incrementing version to v3.6.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 69bddae..699d026 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.6.0' +__version__ = '3.6.1' From 32966fce79a810c8c543cf8e2ded82c4f0591266 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 22 Jun 2023 03:34:17 +0200 Subject: [PATCH 094/132] Fixed several typing issues and added more thorough tests --- _python_utils_tests/test_aio.py | 16 ++- _python_utils_tests/test_containers.py | 10 +- _python_utils_tests/test_decorators.py | 44 ++++++- _python_utils_tests/test_generators.py | 3 +- _python_utils_tests/test_import.py | 6 +- _python_utils_tests/test_logger.py | 1 + _python_utils_tests/test_time.py | 23 +++- pyproject.toml | 7 +- python_utils/aio.py | 31 ++++- python_utils/decorators.py | 75 ++++++++--- python_utils/generators.py | 5 +- python_utils/logger.py | 173 +++++++++++++++++++++---- 12 files changed, 333 insertions(+), 61 deletions(-) diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index 0eb95d3..91d9b99 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -2,8 +2,9 @@ import pytest import asyncio + from python_utils import types -from python_utils.aio import acount +from python_utils.aio import acount, acontainer @pytest.mark.asyncio @@ -20,3 +21,16 @@ async def mock_sleep(delay: float): assert len(sleeps) == 4 assert sum(sleeps) == 4 + + +@pytest.mark.asyncio +async def test_acontainer(): + async def async_gen(): + yield 1 + yield 2 + yield 3 + + assert await acontainer(async_gen) == [1, 2, 3] + assert await acontainer(async_gen()) == [1, 2, 3] + assert await acontainer(async_gen, set) == {1, 2, 3} + assert await acontainer(async_gen(), set) == {1, 2, 3} diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index ba511ae..82352bf 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -3,8 +3,8 @@ from python_utils import containers -def test_unique_list_ignore(): - a = containers.UniqueList() +def test_unique_list_ignore() -> None: + a: containers.UniqueList[int] = containers.UniqueList() a.append(1) a.append(1) assert a == [1] @@ -16,8 +16,10 @@ def test_unique_list_ignore(): a[3] = 5 -def test_unique_list_raise(): - a = containers.UniqueList(*range(20), on_duplicate='raise') +def test_unique_list_raise() -> None: + a: containers.UniqueList[int] = containers.UniqueList( + *range(20), on_duplicate='raise' + ) with pytest.raises(ValueError): a[10:20:2] = [1, 2, 3, 4, 5] diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py index 2ab6c1d..6a67d7a 100644 --- a/_python_utils_tests/test_decorators.py +++ b/_python_utils_tests/test_decorators.py @@ -2,19 +2,19 @@ import pytest -from python_utils.decorators import sample +from python_utils.decorators import sample, wraps_classmethod @pytest.fixture -def random(monkeypatch): +def random(monkeypatch: pytest.MonkeyPatch) -> MagicMock: mock = MagicMock() monkeypatch.setattr( - "python_utils.decorators.random.random", mock, raising=True + 'python_utils.decorators.random.random', mock, raising=True ) return mock -def test_sample_called(random): +def test_sample_called(random: MagicMock): demo_function = MagicMock() decorated = sample(0.5)(demo_function) random.return_value = 0.4 @@ -22,13 +22,13 @@ def test_sample_called(random): random.return_value = 0.0 decorated() args = [1, 2] - kwargs = {"1": 1, "2": 2} + kwargs = {'1': 1, '2': 2} decorated(*args, **kwargs) demo_function.assert_called_with(*args, **kwargs) assert demo_function.call_count == 3 -def test_sample_not_called(random): +def test_sample_not_called(random: MagicMock): demo_function = MagicMock() decorated = sample(0.5)(demo_function) random.return_value = 0.5 @@ -36,3 +36,35 @@ def test_sample_not_called(random): random.return_value = 1.0 decorated() assert demo_function.call_count == 0 + + +class SomeClass: + @classmethod + def some_classmethod(cls, arg): # type: ignore + return arg # type: ignore + + @classmethod + def some_annotated_classmethod(cls, arg: int) -> int: + return arg + + +def test_wraps_classmethod(): # type: ignore + some_class = SomeClass() + some_class.some_classmethod = MagicMock() + wrapped_method = wraps_classmethod( # type: ignore + SomeClass.some_classmethod # type: ignore + )( # type: ignore + some_class.some_classmethod # type: ignore + ) + wrapped_method(123) + some_class.some_classmethod.assert_called_with(123) # type: ignore + + +def test_wraps_classmethod(): # type: ignore + some_class = SomeClass() + some_class.some_annotated_classmethod = MagicMock() + wrapped_method = wraps_classmethod(SomeClass.some_annotated_classmethod)( + some_class.some_annotated_classmethod + ) + wrapped_method(123) # type: ignore + some_class.some_annotated_classmethod.assert_called_with(123) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 2a032e0..39d654b 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -3,6 +3,7 @@ import pytest import python_utils +from python_utils import types @pytest.mark.asyncio @@ -16,7 +17,7 @@ async def test_abatcher(): @pytest.mark.asyncio async def test_abatcher_timed(): - batches = [] + batches: types.List[types.List[int]] = [] async for batch in python_utils.abatcher( python_utils.acount(stop=10, delay=0.08), interval=0.1 ): diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index 2834abc..dcc23d2 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -1,4 +1,4 @@ -from python_utils import import_ +from python_utils import import_, types def test_import_globals_relative_import(): @@ -6,8 +6,8 @@ def test_import_globals_relative_import(): relative_import(i) -def relative_import(level): - locals_ = {} +def relative_import(level: int): + locals_: types.Dict[str, types.Any] = {} globals_ = {'__name__': 'python_utils.import_'} import_.import_global('.formatters', locals_=locals_, globals_=globals_) assert 'camel_to_underscore' in globals_ diff --git a/_python_utils_tests/test_logger.py b/_python_utils_tests/test_logger.py index 7c193b6..1329b56 100644 --- a/_python_utils_tests/test_logger.py +++ b/_python_utils_tests/test_logger.py @@ -15,5 +15,6 @@ class MyClass(Logurud): my_class.info('info') my_class.warning('warning') my_class.error('error') + my_class.critical('critical') my_class.exception('exception') my_class.log(0, 'log') diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 337d4da..acb40a4 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -5,6 +5,7 @@ import pytest import python_utils +from python_utils import types @pytest.mark.parametrize( @@ -25,7 +26,12 @@ ) @pytest.mark.asyncio async def test_aio_timeout_generator( - timeout, interval, interval_multiplier, maximum_interval, iterable, result + timeout: float, + interval: float, + interval_multiplier: float, + maximum_interval: float, + iterable: types.AsyncIterable[types.Any], + result: int, ): i = None async for i in python_utils.aio_timeout_generator( @@ -40,7 +46,7 @@ async def test_aio_timeout_generator( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ (0.01, 0.006, 0.5, 0.01, 'abc', 'c'), - (0.01, 0.006, 0.5, 0.01, itertools.count, 2), + (0.01, 0.006, 0.5, 0.01, itertools.count, 2), # type: ignore (0.01, 0.006, 0.5, 0.01, itertools.count(), 2), (0.01, 0.006, 1.0, None, 'abc', 'c'), ( @@ -48,13 +54,22 @@ async def test_aio_timeout_generator( timedelta(seconds=0.006), 2.0, timedelta(seconds=0.01), - itertools.count, + itertools.count, # type: ignore 2, ), ], ) def test_timeout_generator( - timeout, interval, interval_multiplier, maximum_interval, iterable, result + timeout: float, + interval: float, + interval_multiplier: float, + maximum_interval: float, + iterable: types.Union[ + str, + types.Iterable[types.Any], + types.Callable[..., types.Iterable[types.Any]], + ], + result: int, ): i = None for i in python_utils.timeout_generator( diff --git a/pyproject.toml b/pyproject.toml index 2720bf2..5aaec9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,9 @@ target-version = ['py37', 'py38', 'py39', 'py310', 'py311'] skip-string-normalization = true [tool.pyright] -include = ['python_utils'] -strict = ['python_utils', '_python_utils_tests/test_aio.py'] +# include = ['python_utils'] +include = ['python_utils', '_python_utils_tests'] +strict = ['python_utils', '_python_utils_tests'] # The terminal file is very OS specific and dependent on imports so we're skipping it from type checking ignore = ['python_utils/terminal.py'] -pythonVersion = '3.8' \ No newline at end of file +pythonVersion = '3.8' diff --git a/python_utils/aio.py b/python_utils/aio.py index ef17cb6..00823a8 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -8,6 +8,7 @@ from . import types _N = types.TypeVar('_N', int, float) +_T = types.TypeVar('_T') async def acount( @@ -21,5 +22,33 @@ async def acount( if stop is not None and item >= stop: break - yield types.cast(_N, item) + yield item await asyncio.sleep(delay) + + +async def acontainer( + iterable: types.Union[ + types.AsyncIterable[_T], + types.Callable[..., types.AsyncIterable[_T]], + ], + container: types.Callable[[types.Iterable[_T]], types.Iterable[_T]] = list, +) -> types.Iterable[_T]: + ''' + Asyncio version of list()/set()/tuple()/etc() using an async for loop + + So instead of doing `[item async for item in iterable]` you can do + `await acontainer(iterable)`. + + ''' + iterable_: types.AsyncIterable[_T] + if callable(iterable): + iterable_ = iterable() + else: + iterable_ = iterable + + item: _T + items: types.List[_T] = [] + async for item in iterable_: + items.append(item) + + return container(items) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 6559cf0..1330645 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -3,9 +3,10 @@ import random from . import types -T = types.TypeVar('T') -TC = types.TypeVar('TC', bound=types.Container[types.Any]) -P = types.ParamSpec('P') +_T = types.TypeVar('_T') +_TC = types.TypeVar('_TC', bound=types.Container[types.Any]) +_P = types.ParamSpec('_P') +_S = types.TypeVar('_S', covariant=True) def set_attributes(**kwargs: types.Any) -> types.Callable[..., types.Any]: @@ -33,8 +34,8 @@ def set_attributes(**kwargs: types.Any) -> types.Callable[..., types.Any]: ''' def _set_attributes( - function: types.Callable[P, T] - ) -> types.Callable[P, T]: + function: types.Callable[_P, _T] + ) -> types.Callable[_P, _T]: for key, value in kwargs.items(): setattr(function, key, value) return function @@ -43,11 +44,13 @@ def _set_attributes( def listify( - collection: types.Callable[[types.Iterable[T]], TC] = list, # type: ignore + collection: types.Callable[ + [types.Iterable[_T]], _TC + ] = list, # type: ignore allow_empty: bool = True, ) -> types.Callable[ - [types.Callable[..., types.Optional[types.Iterable[T]]]], - types.Callable[..., TC], + [types.Callable[..., types.Optional[types.Iterable[_T]]]], + types.Callable[..., _TC], ]: ''' Convert any generator to a list or other type of collection. @@ -96,10 +99,10 @@ def listify( ''' def _listify( - function: types.Callable[..., types.Optional[types.Iterable[T]]] - ) -> types.Callable[..., TC]: - def __listify(*args: types.Any, **kwargs: types.Any) -> TC: - result: types.Optional[types.Iterable[T]] = function( + function: types.Callable[..., types.Optional[types.Iterable[_T]]] + ) -> types.Callable[..., _TC]: + def __listify(*args: types.Any, **kwargs: types.Any) -> _TC: + result: types.Optional[types.Iterable[_T]] = function( *args, **kwargs ) if result is None: @@ -134,10 +137,12 @@ def sample(sample_rate: float): ''' def _sample( - function: types.Callable[P, T] - ) -> types.Callable[P, types.Optional[T]]: + function: types.Callable[_P, _T] + ) -> types.Callable[_P, types.Optional[_T]]: @functools.wraps(function) - def __sample(*args: P.args, **kwargs: P.kwargs) -> types.Optional[T]: + def __sample( + *args: _P.args, **kwargs: _P.kwargs + ) -> types.Optional[_T]: if random.random() < sample_rate: return function(*args, **kwargs) else: @@ -152,3 +157,43 @@ def __sample(*args: P.args, **kwargs: P.kwargs) -> types.Optional[T]: return __sample return _sample + + +def wraps_classmethod( + wrapped: types.Callable[types.Concatenate[_S, _P], _T], +) -> types.Callable[ + [ + types.Callable[types.Concatenate[types.Any, _P], _T], + ], + types.Callable[types.Concatenate[types.Type[_S], _P], _T], +]: + ''' + Like `functools.wraps`, but for wrapping classmethods with the type info + from a regular method + ''' + + def _wraps_classmethod( + wrapper: types.Callable[types.Concatenate[types.Any, _P], _T], + ) -> types.Callable[types.Concatenate[types.Type[_S], _P], _T]: + try: # pragma: no cover + wrapper = functools.update_wrapper( + wrapper, + wrapped, + assigned=tuple( + a + for a in functools.WRAPPER_ASSIGNMENTS + if a != '__annotations__' + ), + ) + except AttributeError: + # For some reason `functools.update_wrapper` fails on some test + # runs but not while running actual code + pass + + if annotations := getattr(wrapped, '__annotations__', {}): + annotations.pop('self', None) + wrapper.__annotations__ = annotations + + return wrapper + + return _wraps_classmethod diff --git a/python_utils/generators.py b/python_utils/generators.py index b827ed4..51b76b8 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -9,7 +9,10 @@ async def abatcher( - generator: types.AsyncGenerator[_T, None], + generator: types.Union[ + types.AsyncGenerator[_T, None], + types.AsyncIterator[_T], + ], batch_size: types.Optional[int] = None, interval: types.Optional[types.delta_type] = None, ) -> types.AsyncGenerator[types.List[_T], None]: diff --git a/python_utils/logger.py b/python_utils/logger.py index bcf0bab..9bd80cd 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -1,10 +1,27 @@ import abc -import functools import logging +from . import decorators + __all__ = ['Logged'] -import typing +from . import types + +# From the logging typeshed, converted to be compatible with Python 3.8 +# https://github.com/python/typeshed/blob/main/stdlib/logging/__init__.pyi +_ExcInfoType: types.TypeAlias = types.Union[ + bool, + types.Tuple[ + types.Type[BaseException], + BaseException, + types.Union[types.TracebackType, None], + ], + types.Tuple[None, None, None], + BaseException, + None, +] +_P = types.ParamSpec('_P') +_T = types.TypeVar('_T', covariant=True) class LoggerBase(abc.ABC): @@ -30,7 +47,7 @@ class LoggerBase(abc.ABC): # Being a tad lazy here and not creating a Protocol. # The actual classes define the correct type anyway - logger: typing.Any + logger: types.Any @classmethod def __get_name( # pyright: ignore[reportUnusedFunction] @@ -38,35 +55,147 @@ def __get_name( # pyright: ignore[reportUnusedFunction] ) -> str: return '.'.join(n.strip() for n in name_parts if n.strip()) + @decorators.wraps_classmethod(logging.Logger.debug) @classmethod - @functools.wraps(logging.debug) - def debug(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.debug(msg, *args, **kwargs) + def debug( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: + return cls.logger.debug( + msg, + *args, + exc_info=exc_info, + stack_info=stack_info, + stacklevel=stacklevel, + extra=extra, + ) + + @decorators.wraps_classmethod(logging.Logger.info) + @classmethod + def info( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: + return cls.logger.info( + msg, + *args, + exc_info=exc_info, + stack_info=stack_info, + stacklevel=stacklevel, + extra=extra, + ) + @decorators.wraps_classmethod(logging.Logger.warning) @classmethod - @functools.wraps(logging.info) - def info(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.info(msg, *args, **kwargs) + def warning( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: + return cls.logger.warning( + msg, + *args, + exc_info=exc_info, + stack_info=stack_info, + stacklevel=stacklevel, + extra=extra, + ) + @decorators.wraps_classmethod(logging.Logger.error) @classmethod - @functools.wraps(logging.warning) - def warning(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.warning(msg, *args, **kwargs) + def error( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: + return cls.logger.error( + msg, + *args, + exc_info=exc_info, + stack_info=stack_info, + stacklevel=stacklevel, + extra=extra, + ) + @decorators.wraps_classmethod(logging.Logger.critical) @classmethod - @functools.wraps(logging.error) - def error(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.error(msg, *args, **kwargs) + def critical( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: + return cls.logger.critical( + msg, + *args, + exc_info=exc_info, + stack_info=stack_info, + stacklevel=stacklevel, + extra=extra, + ) + @decorators.wraps_classmethod(logging.Logger.exception) @classmethod - @functools.wraps(logging.exception) - def exception(cls, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.exception(msg, *args, **kwargs) + def exception( + cls, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: + return cls.logger.exception( + msg, + *args, + exc_info=exc_info, + stack_info=stack_info, + stacklevel=stacklevel, + extra=extra, + ) + @decorators.wraps_classmethod(logging.Logger.log) @classmethod - @functools.wraps(logging.log) - def log(cls, lvl: int, msg: str, *args: typing.Any, **kwargs: typing.Any): - cls.logger.log(lvl, msg, *args, **kwargs) + def log( + cls, + level: int, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: + return cls.logger.log( + level, + msg, + *args, + exc_info=exc_info, + stack_info=stack_info, + stacklevel=stacklevel, + extra=extra, + ) class Logged(LoggerBase): @@ -97,7 +226,7 @@ class Logged(LoggerBase): def __get_name(cls, *name_parts: str) -> str: return LoggerBase._LoggerBase__get_name(*name_parts) # type: ignore - def __new__(cls, *args: typing.Any, **kwargs: typing.Any): + def __new__(cls, *args: types.Any, **kwargs: types.Any): cls.logger = logging.getLogger( cls.__get_name(cls.__module__, cls.__name__) ) From 8a2e94fb4604af348a1c5842c610ac671a2765da Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 22 Jun 2023 03:34:21 +0200 Subject: [PATCH 095/132] Incrementing version to v3.7.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 699d026..49296c1 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.6.1' +__version__ = '3.7.0' From da2524d0ef1a6d9b633c6c1c92b287894bce3e3f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 23 Jun 2023 12:24:31 +0200 Subject: [PATCH 096/132] For some reason this test coverage issue occurs on Github actions but not with tox locally... --- python_utils/decorators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 1330645..518f25e 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -185,7 +185,7 @@ def _wraps_classmethod( if a != '__annotations__' ), ) - except AttributeError: + except AttributeError: # pragma: no cover # For some reason `functools.update_wrapper` fails on some test # runs but not while running actual code pass From 26b4680e8b20bdbea1353e196a2023e85c832e7c Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 23 Jun 2023 12:33:14 +0200 Subject: [PATCH 097/132] added return type for strict pyright/mypy --- _python_utils_tests/test_generators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 39d654b..421f2e8 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -16,7 +16,7 @@ async def test_abatcher(): @pytest.mark.asyncio -async def test_abatcher_timed(): +async def test_abatcher_timed() -> None: batches: types.List[types.List[int]] = [] async for batch in python_utils.abatcher( python_utils.acount(stop=10, delay=0.08), interval=0.1 From ed6f27dac7057bc6fca9d55cb77997caf3fe234d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 23 Jun 2023 12:41:09 +0200 Subject: [PATCH 098/132] For some reason the coverage fails, but only on Python 3.9... --- _python_utils_tests/test_aio.py | 6 ++++++ python_utils/aio.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index 91d9b99..8534bac 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -30,7 +30,13 @@ async def async_gen(): yield 2 yield 3 + async def empty_gen(): + if False: + yield 1 + assert await acontainer(async_gen) == [1, 2, 3] assert await acontainer(async_gen()) == [1, 2, 3] assert await acontainer(async_gen, set) == {1, 2, 3} assert await acontainer(async_gen(), set) == {1, 2, 3} + assert await acontainer(empty_gen) == [] + assert await acontainer(empty_gen()) == [] diff --git a/python_utils/aio.py b/python_utils/aio.py index 00823a8..d682e24 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -48,7 +48,7 @@ async def acontainer( item: _T items: types.List[_T] = [] - async for item in iterable_: + async for item in iterable_: # pragma: no branch items.append(item) return container(items) From fef5267f47b164e795c573df26d04206e5cd7a2c Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 21 Aug 2023 02:36:53 +0200 Subject: [PATCH 099/132] Added stale action --- .github/workflows/stale.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..3169ca3 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,20 @@ +name: Close stale issues and pull requests + +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' # Run every day at midnight + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v8 + with: + days-before-stale: 30 + exempt-issue-labels: | + in-progress + help-wanted + pinned + security + enhancement \ No newline at end of file From 2c08cb4781046e3003a3066d0fbb391a3c2aad4a Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 29 Aug 2023 03:09:28 +0200 Subject: [PATCH 100/132] updated stale file --- .github/workflows/stale.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 3169ca3..0740d1a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -12,9 +12,4 @@ jobs: - uses: actions/stale@v8 with: days-before-stale: 30 - exempt-issue-labels: | - in-progress - help-wanted - pinned - security - enhancement \ No newline at end of file + exempt-issue-labels: in-progress,help-wanted,pinned,security,enhancement From 156d0eaae026e023c502aa6e1e6df6eca39d0373 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 29 Aug 2023 23:50:10 +0200 Subject: [PATCH 101/132] updated stale file --- .github/workflows/stale.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 0740d1a..7101b3f 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,3 +13,5 @@ jobs: with: days-before-stale: 30 exempt-issue-labels: in-progress,help-wanted,pinned,security,enhancement + exempt-all-pr-assignees: true + From b868921891fd5ba59f00e8ee29d436a17a41963e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 25 Sep 2023 01:30:33 +0200 Subject: [PATCH 102/132] Added slicable deque --- _python_utils_tests/test_containers.py | 42 ++++++++++++++++++++++++++ _python_utils_tests/test_decorators.py | 2 +- python_utils/containers.py | 40 +++++++++++++++++++----- python_utils/decorators.py | 10 +++--- 4 files changed, 80 insertions(+), 14 deletions(-) diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index 82352bf..e112f9b 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -29,3 +29,45 @@ def test_unique_list_raise() -> None: del a[10] del a[5:15] + + +def test_sliceable_deque() -> None: + d: containers.SlicableDeque[int] = containers.SlicableDeque(range(10)) + assert d[0] == 0 + assert d[-1] == 9 + assert d[1:3] == [1, 2] + assert d[1:3:2] == [1] + assert d[1:3:-1] == [] + assert d[3:1] == [] + assert d[3:1:-1] == [3, 2] + assert d[3:1:-2] == [3] + with pytest.raises(ValueError): + assert d[1:3:0] + assert d[1:3:1] == [1, 2] + assert d[1:3:2] == [1] + assert d[1:3:-1] == [] + + +def test_sliceable_deque_pop() -> None: + d: containers.SlicableDeque[int] = containers.SlicableDeque(range(10)) + + assert d.pop() == 9 == 9 + assert d.pop(0) == 0 + + with pytest.raises(IndexError): + assert d.pop(100) + + with pytest.raises(IndexError): + assert d.pop(2) + + with pytest.raises(IndexError): + assert d.pop(-2) + + +def test_sliceable_deque_eq() -> None: + d: containers.SlicableDeque[int] = containers.SlicableDeque([1, 2, 3]) + assert d == [1, 2, 3] + assert d == (1, 2, 3) + assert d == {1, 2, 3} + assert d == d + assert d == containers.SlicableDeque([1, 2, 3]) diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py index 6a67d7a..16d40e0 100644 --- a/_python_utils_tests/test_decorators.py +++ b/_python_utils_tests/test_decorators.py @@ -60,7 +60,7 @@ def test_wraps_classmethod(): # type: ignore some_class.some_classmethod.assert_called_with(123) # type: ignore -def test_wraps_classmethod(): # type: ignore +def test_wraps_annotated_classmethod(): # type: ignore some_class = SomeClass() some_class.some_annotated_classmethod = MagicMock() wrapped_method = wraps_classmethod(SomeClass.some_annotated_classmethod)( diff --git a/python_utils/containers.py b/python_utils/containers.py index 4685ab9..7984ace 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,7 +1,7 @@ # pyright: reportIncompatibleMethodOverride=false import abc -import typing import collections +import typing from . import types @@ -238,7 +238,7 @@ def __init__( def insert(self, index: types.SupportsIndex, value: HT) -> None: if value in self._set: if self.on_duplicate == 'raise': - raise ValueError('Duplicate value: %s' % value) + raise ValueError(f'Duplicate value: {value}') else: return @@ -248,7 +248,7 @@ def insert(self, index: types.SupportsIndex, value: HT) -> None: def append(self, value: HT) -> None: if value in self._set: if self.on_duplicate == 'raise': - raise ValueError('Duplicate value: %s' % value) + raise ValueError(f'Duplicate value: {value}') else: return @@ -258,11 +258,11 @@ def append(self, value: HT) -> None: def __contains__(self, item: HT) -> bool: # type: ignore return item in self._set - @types.overload + @typing.overload def __setitem__(self, indices: types.SupportsIndex, values: HT) -> None: ... - @types.overload + @typing.overload def __setitem__(self, indices: slice, values: types.Iterable[HT]) -> None: ... @@ -310,12 +310,14 @@ def __delitem__( super().__delitem__(index) +# Type hinting `collections.deque` does not work consistently between Python +# runtime, mypy and pyright currently so we have to ignore the errors class SlicableDeque(types.Generic[T], collections.deque): # type: ignore - @types.overload + @typing.overload def __getitem__(self, index: types.SupportsIndex) -> T: ... - @types.overload + @typing.overload def __getitem__(self, index: slice) -> 'SlicableDeque[T]': ... @@ -340,6 +342,30 @@ def __getitem__( else: return types.cast(T, super().__getitem__(index)) + def __eq__(self, other: types.Any) -> bool: + # Allow for comparison with a list or tuple + if isinstance(other, list): + return list(self) == other + elif isinstance(other, tuple): + return tuple(self) == other + elif isinstance(other, set): + return set(self) == other + else: + return super().__eq__(other) + + def pop(self, index: int = -1) -> T: + # We need to allow for an index but a deque only allows the removal of + # the first or last item. + if index == 0: + return typing.cast(T, super().popleft()) + elif index in {-1, len(self) - 1}: + return typing.cast(T, super().pop()) + else: + raise IndexError( + 'Only index 0 and the last index (`N-1` or `-1`) ' + 'are supported' + ) + if __name__ == '__main__': import doctest diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 518f25e..8bea846 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -1,3 +1,4 @@ +import contextlib import functools import logging import random @@ -175,7 +176,9 @@ def wraps_classmethod( def _wraps_classmethod( wrapper: types.Callable[types.Concatenate[types.Any, _P], _T], ) -> types.Callable[types.Concatenate[types.Type[_S], _P], _T]: - try: # pragma: no cover + # For some reason `functools.update_wrapper` fails on some test + # runs but not while running actual code + with contextlib.suppress(AttributeError): wrapper = functools.update_wrapper( wrapper, wrapped, @@ -185,11 +188,6 @@ def _wraps_classmethod( if a != '__annotations__' ), ) - except AttributeError: # pragma: no cover - # For some reason `functools.update_wrapper` fails on some test - # runs but not while running actual code - pass - if annotations := getattr(wrapped, '__annotations__', {}): annotations.pop('self', None) wrapper.__annotations__ = annotations From 7e390e2f4492007bf192347dfd283f9d810d768f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 25 Sep 2023 01:30:50 +0200 Subject: [PATCH 103/132] Incrementing version to v3.8.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 49296c1..33d0708 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.7.0' +__version__ = '3.8.0' From 6fd370408c274dc2fb0c1f3f67863166192fa996 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 26 Sep 2023 03:46:40 +0200 Subject: [PATCH 104/132] fixed silly typo --- _python_utils_tests/test_containers.py | 8 ++++---- python_utils/containers.py | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/_python_utils_tests/test_containers.py b/_python_utils_tests/test_containers.py index e112f9b..a38609d 100644 --- a/_python_utils_tests/test_containers.py +++ b/_python_utils_tests/test_containers.py @@ -32,7 +32,7 @@ def test_unique_list_raise() -> None: def test_sliceable_deque() -> None: - d: containers.SlicableDeque[int] = containers.SlicableDeque(range(10)) + d: containers.SliceableDeque[int] = containers.SliceableDeque(range(10)) assert d[0] == 0 assert d[-1] == 9 assert d[1:3] == [1, 2] @@ -49,7 +49,7 @@ def test_sliceable_deque() -> None: def test_sliceable_deque_pop() -> None: - d: containers.SlicableDeque[int] = containers.SlicableDeque(range(10)) + d: containers.SliceableDeque[int] = containers.SliceableDeque(range(10)) assert d.pop() == 9 == 9 assert d.pop(0) == 0 @@ -65,9 +65,9 @@ def test_sliceable_deque_pop() -> None: def test_sliceable_deque_eq() -> None: - d: containers.SlicableDeque[int] = containers.SlicableDeque([1, 2, 3]) + d: containers.SliceableDeque[int] = containers.SliceableDeque([1, 2, 3]) assert d == [1, 2, 3] assert d == (1, 2, 3) assert d == {1, 2, 3} assert d == d - assert d == containers.SlicableDeque([1, 2, 3]) + assert d == containers.SliceableDeque([1, 2, 3]) diff --git a/python_utils/containers.py b/python_utils/containers.py index 7984ace..19b91c4 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -312,28 +312,28 @@ def __delitem__( # Type hinting `collections.deque` does not work consistently between Python # runtime, mypy and pyright currently so we have to ignore the errors -class SlicableDeque(types.Generic[T], collections.deque): # type: ignore +class SliceableDeque(types.Generic[T], collections.deque): # type: ignore @typing.overload def __getitem__(self, index: types.SupportsIndex) -> T: ... @typing.overload - def __getitem__(self, index: slice) -> 'SlicableDeque[T]': + def __getitem__(self, index: slice) -> 'SliceableDeque[T]': ... def __getitem__( self, index: types.Union[types.SupportsIndex, slice] - ) -> types.Union[T, 'SlicableDeque[T]']: + ) -> types.Union[T, 'SliceableDeque[T]']: ''' Return the item or slice at the given index. - >>> d = SlicableDeque[int]([1, 2, 3, 4, 5]) + >>> d = SliceableDeque[int]([1, 2, 3, 4, 5]) >>> d[1:4] - SlicableDeque([2, 3, 4]) + SliceableDeque([2, 3, 4]) - >>> d = SlicableDeque[str](['a', 'b', 'c']) + >>> d = SliceableDeque[str](['a', 'b', 'c']) >>> d[-2:] - SlicableDeque(['b', 'c']) + SliceableDeque(['b', 'c']) ''' if isinstance(index, slice): From e3e7d4038d4a8ac5ca02045069f59a18a2688e41 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 26 Sep 2023 03:46:58 +0200 Subject: [PATCH 105/132] Incrementing version to v3.8.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 33d0708..46292c0 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.8.0' +__version__ = '3.8.1' From 7f82d15b9416ed383c6d4e5e91f0671cf906f92f Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 25 Jan 2024 03:46:59 +0100 Subject: [PATCH 106/132] Windows can also return an OSError instead of a ValueError for a `fromtimestamp` overflow --- python_utils/time.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/time.py b/python_utils/time.py index 4084a5a..51cacc1 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -130,7 +130,7 @@ def format_time( try: # pragma: no cover dt = datetime.datetime.fromtimestamp(seconds) - except ValueError: # pragma: no cover + except (ValueError, OSError): # pragma: no cover dt = datetime.datetime.max return str(dt) elif isinstance(timestamp, datetime.date): From 26a2c3056f210fc381354fbc16e688cabfb7a6ac Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Thu, 25 Jan 2024 10:19:36 +0100 Subject: [PATCH 107/132] Incrementing version to v3.8.2 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 46292c0..025b7a7 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -7,4 +7,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.8.1' +__version__ = '3.8.2' From f6d7e13b9f1d037046a9789c0cf70fcd25b12214 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 3 Mar 2024 14:49:57 +0100 Subject: [PATCH 108/132] Create .readthedocs.yaml --- .readthedocs.yaml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .readthedocs.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..bee434d --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,35 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + # You can also specify other tool versions: + # nodejs: "20" + # rust: "1.70" + # golang: "1.20" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs + # builder: "dirhtml" + # Fail on all warnings to avoid broken references + # fail_on_warning: true + +# Optionally build your docs in additional formats such as PDF and ePub +formats: + - pdf + - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: docs/requirements.txt From bbbc4e56e4d9ade63137d765a49f038e4cdc35aa Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 20 Sep 2024 21:38:59 +0200 Subject: [PATCH 109/132] Many code quality improvements, ruff compliance, pyright strict compliance, dropped Python 3.8 --- _python_utils_tests/test_aio.py | 29 ++++-- _python_utils_tests/test_generators.py | 2 +- _python_utils_tests/test_import.py | 8 +- _python_utils_tests/test_logger.py | 1 - _python_utils_tests/test_time.py | 26 ++++-- docs/conf.py | 22 ++--- pyproject.toml | 7 +- python_utils/__about__.py | 12 +++ python_utils/__init__.py | 52 +++++++++++ python_utils/aio.py | 75 ++++++++++++++-- python_utils/compat.py | 0 python_utils/containers.py | 120 +++++++++++++++++++++---- python_utils/converters.py | 66 +++++++------- python_utils/decorators.py | 35 ++++---- python_utils/exceptions.py | 4 +- python_utils/formatters.py | 25 +++--- python_utils/generators.py | 9 +- python_utils/import_.py | 28 +++--- python_utils/logger.py | 12 +-- python_utils/loguru.py | 1 + python_utils/terminal.py | 27 +++--- python_utils/time.py | 54 ++++++----- python_utils/types.py | 18 ++-- setup.py | 16 ++-- tox.ini | 7 +- 25 files changed, 462 insertions(+), 194 deletions(-) delete mode 100644 python_utils/compat.py diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index 8534bac..db3ffa5 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -1,10 +1,9 @@ -from datetime import datetime -import pytest import asyncio +import pytest from python_utils import types -from python_utils.aio import acount, acontainer +from python_utils.aio import acontainer, acount, adict @pytest.mark.asyncio @@ -16,8 +15,8 @@ async def mock_sleep(delay: float): monkeypatch.setattr(asyncio, 'sleep', mock_sleep) - async for i in acount(delay=1, stop=3.5): - print('i', i, datetime.now()) + async for _i in acount(delay=1, stop=3.5): + pass assert len(sleeps) == 4 assert sum(sleeps) == 4 @@ -38,5 +37,25 @@ async def empty_gen(): assert await acontainer(async_gen()) == [1, 2, 3] assert await acontainer(async_gen, set) == {1, 2, 3} assert await acontainer(async_gen(), set) == {1, 2, 3} + assert await acontainer(async_gen, list) == [1, 2, 3] + assert await acontainer(async_gen(), list) == [1, 2, 3] + assert await acontainer(async_gen, tuple) == (1, 2, 3) + assert await acontainer(async_gen(), tuple) == (1, 2, 3) assert await acontainer(empty_gen) == [] assert await acontainer(empty_gen()) == [] + assert await acontainer(empty_gen, set) == set() + assert await acontainer(empty_gen(), set) == set() + assert await acontainer(empty_gen, list) == list() + assert await acontainer(empty_gen(), list) == list() + assert await acontainer(empty_gen, tuple) == tuple() + assert await acontainer(empty_gen(), tuple) == tuple() + + +@pytest.mark.asyncio +async def test_adict(): + async def async_gen(): + yield 1, 2 + yield 3, 4 + yield 5, 6 + + assert await adict(async_gen) == {1: 2, 3: 4, 5: 6} diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 421f2e8..9a80795 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -63,6 +63,6 @@ def test_batcher(): assert len(batch) == 3 for batch in python_utils.batcher(range(4), 3): - pass + assert batch is not None assert len(batch) == 1 diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index dcc23d2..9de85ba 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -14,8 +14,8 @@ def relative_import(level: int): def test_import_globals_without_inspection(): - locals_ = {} - globals_ = {'__name__': __name__} + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} import_.import_global( 'python_utils.formatters', locals_=locals_, globals_=globals_ ) @@ -23,8 +23,8 @@ def test_import_globals_without_inspection(): def test_import_globals_single_method(): - locals_ = {} - globals_ = {'__name__': __name__} + locals_: types.Dict[str, types.Any] = {} + globals_: types.Dict[str, types.Any] = {'__name__': __name__} import_.import_global( 'python_utils.formatters', ['camel_to_underscore'], diff --git a/_python_utils_tests/test_logger.py b/_python_utils_tests/test_logger.py index 1329b56..a443603 100644 --- a/_python_utils_tests/test_logger.py +++ b/_python_utils_tests/test_logger.py @@ -2,7 +2,6 @@ from python_utils.loguru import Logurud - loguru = pytest.importorskip('loguru') diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index acb40a4..ffd1de5 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -79,7 +79,7 @@ def test_timeout_generator( iterable=iterable, maximum_interval=maximum_interval, ): - pass + assert i is not None assert i == result @@ -123,10 +123,7 @@ async def generator(): @pytest.mark.asyncio -async def test_aio_generator_timeout_detector_decorator(): - # Make pyright happy - i = None - +async def test_aio_generator_timeout_detector_decorator_reraise(): # Test regular timeout with reraise @python_utils.aio_generator_timeout_detector_decorator(timeout=0.05) async def generator_timeout(): @@ -135,9 +132,15 @@ async def generator_timeout(): yield i with pytest.raises(asyncio.TimeoutError): - async for i in generator_timeout(): + async for _ in generator_timeout(): pass + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_decorator_clean_exit(): + # Make pyright happy + i = None + # Test regular timeout with clean exit @python_utils.aio_generator_timeout_detector_decorator( timeout=0.05, on_timeout=None @@ -152,6 +155,9 @@ async def generator_clean(): assert i == 4 + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_decorator_reraise_total(): # Test total timeout with reraise @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.1) async def generator_reraise(): @@ -160,9 +166,15 @@ async def generator_reraise(): yield i with pytest.raises(asyncio.TimeoutError): - async for i in generator_reraise(): + async for _ in generator_reraise(): pass + +@pytest.mark.asyncio +async def test_aio_generator_timeout_detector_decorator_clean_total(): + # Make pyright happy + i = None + # Test total timeout with clean exit @python_utils.aio_generator_timeout_detector_decorator( total_timeout=0.1, on_timeout=None diff --git a/docs/conf.py b/docs/conf.py index 63e2a44..1fcd968 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,18 +1,20 @@ -# Configuration file for the Sphinx documentation builder. +""" +Configuration file for the Sphinx documentation builder. # -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html +This file only contains a selection of the most common options. For a full +list see the documentation: +https://www.sphinx-doc.org/en/master/usage/configuration.html -# -- Path setup -------------------------------------------------------------- +-- Path setup -------------------------------------------------------------- -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. +If extensions (or modules to document with autodoc) are in another directory, +add these directories to sys.path here. If the directory is relative to the +documentation root, use os.path.abspath to make it absolute, like shown here. # -from datetime import date +""" import os import sys +from datetime import date sys.path.insert(0, os.path.abspath('..')) @@ -27,7 +29,6 @@ # The full version, including alpha/beta/rc tags release = __about__.__version__ - # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be @@ -50,7 +51,6 @@ # This pattern also affects html_static_path and html_extra_path. exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for diff --git a/pyproject.toml b/pyproject.toml index 5aaec9e..4f8224e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,8 +5,9 @@ skip-string-normalization = true [tool.pyright] # include = ['python_utils'] -include = ['python_utils', '_python_utils_tests'] -strict = ['python_utils', '_python_utils_tests'] +include = ['python_utils', '_python_utils_tests', 'setup.py'] +strict = ['python_utils', '_python_utils_tests', 'setup.py'] # The terminal file is very OS specific and dependent on imports so we're skipping it from type checking ignore = ['python_utils/terminal.py'] -pythonVersion = '3.8' +pythonVersion = '3.9' + diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 025b7a7..4689e2b 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -1,3 +1,15 @@ +""" +This module contains metadata about the `python-utils` package. + +Attributes: + __package_name__ (str): The name of the package. + __author__ (str): The author of the package. + __author_email__ (str): The email of the author. + __description__ (str): A brief description of the package. + __url__ (str): The URL of the package's repository. + __version__ (str): The current version of the package. +""" + __package_name__: str = 'python-utils' __author__: str = 'Rick van Hattem' __author_email__: str = 'Wolph@wol.ph' diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 900bcc8..e9fd9b1 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -1,3 +1,55 @@ +""" +This module initializes the `python_utils` package by importing various +submodules and functions. + +Submodules: + aio + compat + converters + decorators + formatters + generators + import_ + logger + terminal + time + types + +Functions: + acount + remap + scale_1024 + to_float + to_int + to_str + to_unicode + listify + set_attributes + raise_exception + reraise + camel_to_underscore + timesince + abatcher + batcher + import_global + get_terminal_size + aio_generator_timeout_detector + aio_generator_timeout_detector_decorator + aio_timeout_generator + delta_to_seconds + delta_to_seconds_or_none + format_time + timedelta_to_seconds + timeout_generator + +Classes: + CastedDict + LazyCastedDict + UniqueList + Logged + LoggerBase +""" + from . import ( aio, compat, diff --git a/python_utils/aio.py b/python_utils/aio.py index d682e24..0146c54 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -1,14 +1,15 @@ -''' -Asyncio equivalents to regular Python functions. +"""Asyncio equivalents to regular Python functions.""" -''' import asyncio import itertools +import typing from . import types _N = types.TypeVar('_N', int, float) _T = types.TypeVar('_T') +_K = types.TypeVar('_K') +_V = types.TypeVar('_V') async def acount( @@ -17,7 +18,7 @@ async def acount( delay: float = 0, stop: types.Optional[_N] = None, ) -> types.AsyncIterator[_N]: - '''Asyncio version of itertools.count()''' + """Asyncio version of itertools.count().""" for item in itertools.count(start, step): # pragma: no branch if stop is not None and item >= stop: break @@ -26,6 +27,36 @@ async def acount( await asyncio.sleep(delay) +@typing.overload +async def acontainer( + iterable: types.Union[ + types.AsyncIterable[_T], + types.Callable[..., types.AsyncIterable[_T]], + ], + container: types.Type[types.Tuple[_T, ...]], +) -> types.Tuple[_T, ...]: ... + + +@typing.overload +async def acontainer( + iterable: types.Union[ + types.AsyncIterable[_T], + types.Callable[..., types.AsyncIterable[_T]], + ], + container: types.Type[types.List[_T]] = list, +) -> types.List[_T]: ... + + +@typing.overload +async def acontainer( + iterable: types.Union[ + types.AsyncIterable[_T], + types.Callable[..., types.AsyncIterable[_T]], + ], + container: types.Type[types.Set[_T]], +) -> types.Set[_T]: ... + + async def acontainer( iterable: types.Union[ types.AsyncIterable[_T], @@ -33,13 +64,13 @@ async def acontainer( ], container: types.Callable[[types.Iterable[_T]], types.Iterable[_T]] = list, ) -> types.Iterable[_T]: - ''' - Asyncio version of list()/set()/tuple()/etc() using an async for loop + """ + Asyncio version of list()/set()/tuple()/etc() using an async for loop. So instead of doing `[item async for item in iterable]` you can do `await acontainer(iterable)`. - ''' + """ iterable_: types.AsyncIterable[_T] if callable(iterable): iterable_ = iterable() @@ -52,3 +83,33 @@ async def acontainer( items.append(item) return container(items) + + +async def adict( + iterable: types.Union[ + types.AsyncIterable[types.Tuple[_K, _V]], + types.Callable[..., types.AsyncIterable[types.Tuple[_K, _V]]], + ], + container: types.Callable[ + [types.Iterable[types.Tuple[_K, _V]]], types.Mapping[_K, _V] + ] = dict, +) -> types.Mapping[_K, _V]: + """ + Asyncio version of dict() using an async for loop. + + So instead of doing `{key: value async for key, value in iterable}` you + can do `await adict(iterable)`. + + """ + iterable_: types.AsyncIterable[types.Tuple[_K, _V]] + if callable(iterable): + iterable_ = iterable() + else: + iterable_ = iterable + + item: types.Tuple[_K, _V] + items: types.List[types.Tuple[_K, _V]] = [] + async for item in iterable_: + items.append(item) + + return container(items) diff --git a/python_utils/compat.py b/python_utils/compat.py deleted file mode 100644 index e69de29..0000000 diff --git a/python_utils/containers.py b/python_utils/containers.py index 19b91c4..1809c7d 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -1,3 +1,58 @@ +""" +This module provides custom container classes with enhanced functionality. + +Classes: + CastedDictBase: Abstract base class for dictionaries that cast keys and + values. + CastedDict: Dictionary that casts keys and values to specified types. + LazyCastedDict: Dictionary that lazily casts values to specified types upon + access. + UniqueList: List that only allows unique values, with configurable behavior + on duplicates. + SliceableDeque: Deque that supports slicing and enhanced equality checks. + +Type Aliases: + KT: Type variable for dictionary keys. + VT: Type variable for dictionary values. + DT: Type alias for a dictionary with keys of type KT and values of type VT. + KT_cast: Type alias for a callable that casts dictionary keys. + VT_cast: Type alias for a callable that casts dictionary values. + HT: Type variable for hashable values in UniqueList. + T: Type variable for generic types. + DictUpdateArgs: Union type for arguments that can be used to update a + dictionary. + OnDuplicate: Literal type for handling duplicate values in UniqueList. + +Usage: + - CastedDict and LazyCastedDict can be used to create dictionaries with + automatic type casting. + - UniqueList ensures all elements are unique and can raise an error on + duplicates. + - SliceableDeque extends deque with slicing support and enhanced equality + checks. + +Examples: + >>> d = CastedDict(int, int) + >>> d[1] = 2 + >>> d['3'] = '4' + >>> d.update({'5': '6'}) + >>> d.update([('7', '8')]) + >>> d + {1: 2, 3: 4, 5: 6, 7: 8} + + >>> l = UniqueList(1, 2, 3) + >>> l.append(4) + >>> l.append(4) + >>> l.insert(0, 4) + >>> l.insert(0, 5) + >>> l[1] = 10 + >>> l + [5, 10, 2, 3, 4] + + >>> d = SliceableDeque([1, 2, 3, 4, 5]) + >>> d[1:4] + SliceableDeque([2, 3, 4]) +""" # pyright: reportIncompatibleMethodOverride=false import abc import collections @@ -35,6 +90,25 @@ class CastedDictBase(types.Dict[KT, VT], abc.ABC): + """ + Abstract base class for dictionaries that cast keys and values. + + Attributes: + _key_cast (KT_cast[KT]): Callable to cast dictionary keys. + _value_cast (VT_cast[VT]): Callable to cast dictionary values. + + Methods: + __init__(key_cast: KT_cast[KT] = None, value_cast: VT_cast[VT] = None, + *args: DictUpdateArgs[KT, VT], **kwargs: VT) -> None: + Initializes the dictionary with optional key and value casting + callables. + update(*args: DictUpdateArgs[types.Any, types.Any], + **kwargs: types.Any) -> None: + Updates the dictionary with the given arguments. + __setitem__(key: types.Any, value: types.Any) -> None: + Sets the item in the dictionary, casting the key if a key cast + callable is provided. + """ _key_cast: KT_cast[KT] _value_cast: VT_cast[VT] @@ -45,6 +119,20 @@ def __init__( *args: DictUpdateArgs[KT, VT], **kwargs: VT, ) -> None: + """ + Initializes the CastedDictBase with optional key and value + casting callables. + + Args: + key_cast (KT_cast[KT], optional): Callable to cast + dictionary keys. Defaults to None. + value_cast (VT_cast[VT], optional): Callable to cast + dictionary values. Defaults to None. + *args (DictUpdateArgs[KT, VT]): Arguments to initialize + the dictionary. + **kwargs (VT): Keyword arguments to initialize the + dictionary. + """ self._value_cast = value_cast self._key_cast = key_cast self.update(*args, **kwargs) @@ -67,7 +155,7 @@ def __setitem__(self, key: types.Any, value: types.Any) -> None: class CastedDict(CastedDictBase[KT, VT]): - ''' + """ Custom dictionary that casts keys and values to the specified typing. Note that you can specify the types for mypy and type hinting with: @@ -99,7 +187,7 @@ class CastedDict(CastedDictBase[KT, VT]): >>> d.update([('7', '8')]) >>> d {1: 2, '3': '4', '5': '6', '7': '8'} - ''' + """ def __setitem__(self, key: typing.Any, value: typing.Any) -> None: if self._value_cast is not None: @@ -109,7 +197,7 @@ def __setitem__(self, key: typing.Any, value: typing.Any) -> None: class LazyCastedDict(CastedDictBase[KT, VT]): - ''' + """ Custom dictionary that casts keys and lazily casts values to the specified typing. Note that the values are cast only when they are accessed and are not cached between executions. @@ -152,7 +240,7 @@ class LazyCastedDict(CastedDictBase[KT, VT]): [(1, 2), ('3', '4'), ('5', '6'), ('7', '8')] >>> d['3'] '4' - ''' + """ def __setitem__(self, key: types.Any, value: types.Any): if self._key_cast is not None: @@ -189,7 +277,7 @@ def values(self) -> types.Generator[VT, None, None]: # type: ignore class UniqueList(types.List[HT]): - ''' + """ A list that only allows unique values. Duplicate values are ignored by default, but can be configured to raise an exception instead. @@ -220,7 +308,7 @@ class UniqueList(types.List[HT]): Traceback (most recent call last): ... ValueError: Duplicate value: 4 - ''' + """ _set: types.Set[HT] @@ -259,12 +347,14 @@ def __contains__(self, item: HT) -> bool: # type: ignore return item in self._set @typing.overload - def __setitem__(self, indices: types.SupportsIndex, values: HT) -> None: - ... + def __setitem__( + self, indices: types.SupportsIndex, values: HT + ) -> None: ... @typing.overload - def __setitem__(self, indices: slice, values: types.Iterable[HT]) -> None: - ... + def __setitem__( + self, indices: slice, values: types.Iterable[HT] + ) -> None: ... def __setitem__( self, @@ -314,17 +404,15 @@ def __delitem__( # runtime, mypy and pyright currently so we have to ignore the errors class SliceableDeque(types.Generic[T], collections.deque): # type: ignore @typing.overload - def __getitem__(self, index: types.SupportsIndex) -> T: - ... + def __getitem__(self, index: types.SupportsIndex) -> T: ... @typing.overload - def __getitem__(self, index: slice) -> 'SliceableDeque[T]': - ... + def __getitem__(self, index: slice) -> 'SliceableDeque[T]': ... def __getitem__( self, index: types.Union[types.SupportsIndex, slice] ) -> types.Union[T, 'SliceableDeque[T]']: - ''' + """ Return the item or slice at the given index. >>> d = SliceableDeque[int]([1, 2, 3, 4, 5]) @@ -335,7 +423,7 @@ def __getitem__( >>> d[-2:] SliceableDeque(['b', 'c']) - ''' + """ if isinstance(index, slice): start, stop, step = index.indices(len(self)) return self.__class__(self[i] for i in range(start, stop, step)) diff --git a/python_utils/converters.py b/python_utils/converters.py index 68438ee..da221c5 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -1,21 +1,26 @@ +from __future__ import annotations + import decimal import math import re -import typing +from typing import Union from . import types _TN = types.TypeVar('_TN', bound=types.DecimalNumber) +_RegexpType: types.TypeAlias = Union[ + types.Pattern[str], str, types.Literal[True], None] + def to_int( - input_: typing.Optional[str] = None, + input_: str | None = None, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.Optional[types.Pattern[str]] = None, + regexp: _RegexpType = None, ) -> int: - r''' - Convert the given input to an integer or return default + r""" + Convert the given input to an integer or return default. When trying to convert the exceptions given in the exception parameter are automatically catched and the default will be returned. @@ -74,7 +79,7 @@ def to_int( Traceback (most recent call last): ... TypeError: unknown argument for regexp parameter: 123 - ''' + """ if regexp is True: regexp = re.compile(r'(\d+)') elif isinstance(regexp, str): @@ -82,12 +87,11 @@ def to_int( elif hasattr(regexp, 'search'): pass elif regexp is not None: - raise TypeError('unknown argument for regexp parameter: %r' % regexp) + raise TypeError(f'unknown argument for regexp parameter: {regexp!r}') try: - if regexp and input_: - if match := regexp.search(input_): - input_ = match.groups()[-1] + if regexp and input_ and (match := regexp.search(input_)): + input_ = match.groups()[-1] if input_ is None: return default @@ -101,10 +105,10 @@ def to_float( input_: str, default: int = 0, exception: types.ExceptionsType = (ValueError, TypeError), - regexp: types.Optional[types.Pattern[str]] = None, + regexp: _RegexpType = None, ) -> types.Number: - r''' - Convert the given `input_` to an integer or return default + r""" + Convert the given `input_` to an integer or return default. When trying to convert the exceptions given in the exception parameter are automatically catched and the default will be returned. @@ -153,8 +157,7 @@ def to_float( Traceback (most recent call last): ... TypeError: unknown argument for regexp parameter - ''' - + """ if regexp is True: regexp = re.compile(r'(\d+(\.\d+|))') elif isinstance(regexp, str): @@ -165,9 +168,8 @@ def to_float( raise TypeError('unknown argument for regexp parameter') try: - if regexp: - if match := regexp.search(input_): - input_ = match.group(1) + if regexp and (match := regexp.search(input_)): + input_ = match.group(1) return float(input_) except exception: return default @@ -178,7 +180,7 @@ def to_unicode( encoding: str = 'utf-8', errors: str = 'replace', ) -> str: - '''Convert objects to unicode, if needed decodes string with the given + """Convert objects to unicode, if needed decodes string with the given encoding and errors settings. :rtype: str @@ -187,14 +189,15 @@ def to_unicode( 'a' >>> to_unicode('a') 'a' - >>> to_unicode(u'a') + >>> to_unicode('a') 'a' - >>> class Foo(object): __str__ = lambda s: u'a' + >>> class Foo(object): + ... __str__ = lambda s: 'a' >>> to_unicode(Foo()) 'a' >>> to_unicode(Foo) "" - ''' + """ if isinstance(input_, bytes): input_ = input_.decode(encoding, errors) else: @@ -207,22 +210,23 @@ def to_str( encoding: str = 'utf-8', errors: str = 'replace', ) -> bytes: - '''Convert objects to string, encodes to the given encoding + """Convert objects to string, encodes to the given encoding. :rtype: str >>> to_str('a') b'a' - >>> to_str(u'a') + >>> to_str('a') b'a' >>> to_str(b'a') b'a' - >>> class Foo(object): __str__ = lambda s: u'a' + >>> class Foo(object): + ... __str__ = lambda s: 'a' >>> to_str(Foo()) 'a' >>> to_str(Foo) "" - ''' + """ if not isinstance(input_, bytes): if not hasattr(input_, 'encode'): input_ = str(input_) @@ -235,7 +239,7 @@ def scale_1024( x: types.Number, n_prefixes: int, ) -> types.Tuple[types.Number, types.Number]: - '''Scale a number down to a suitable size, based on powers of 1024. + """Scale a number down to a suitable size, based on powers of 1024. Returns the scaled number and the power of 1024 used. @@ -251,7 +255,7 @@ def scale_1024( (0.5, 0) >>> scale_1024(1, 2) (1.0, 0) - ''' + """ if x <= 0: power = 0 else: @@ -267,7 +271,7 @@ def remap( new_min: _TN, new_max: _TN, ) -> _TN: - ''' + """ remap a value from one range into another. >>> remap(500, 0, 1000, 0, 100) @@ -338,7 +342,7 @@ def remap( the returned type will be `int`. :rtype: int, float, decimal.Decimal - ''' + """ type_: types.Type[types.DecimalNumber] if ( isinstance(value, decimal.Decimal) @@ -379,7 +383,7 @@ def remap( new_value = (value - old_min) * new_range # type: ignore - if type_ == int: + if type_ is int: new_value //= old_range # type: ignore else: new_value /= old_range # type: ignore diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 8bea846..0459f88 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -2,6 +2,7 @@ import functools import logging import random + from . import types _T = types.TypeVar('_T') @@ -11,7 +12,7 @@ def set_attributes(**kwargs: types.Any) -> types.Callable[..., types.Any]: - '''Decorator to set attributes on functions and classes + """Decorator to set attributes on functions and classes. A common usage for this pattern is the Django Admin where functions can get an optional short_description. To illustrate: @@ -23,19 +24,19 @@ def set_attributes(**kwargs: types.Any) -> types.Callable[..., types.Any]: >>> @set_attributes(short_description='Name') ... def upper_case_name(self, obj): - ... return ("%s %s" % (obj.first_name, obj.last_name)).upper() + ... return ('%s %s' % (obj.first_name, obj.last_name)).upper() The standard Django version: >>> def upper_case_name(obj): - ... return ("%s %s" % (obj.first_name, obj.last_name)).upper() + ... return ('%s %s' % (obj.first_name, obj.last_name)).upper() >>> upper_case_name.short_description = 'Name' - ''' + """ def _set_attributes( - function: types.Callable[_P, _T] + function: types.Callable[_P, _T], ) -> types.Callable[_P, _T]: for key, value in kwargs.items(): setattr(function, key, value) @@ -45,15 +46,13 @@ def _set_attributes( def listify( - collection: types.Callable[ - [types.Iterable[_T]], _TC - ] = list, # type: ignore + collection: types.Callable[[types.Iterable[_T]], _TC] = list, # type: ignore allow_empty: bool = True, ) -> types.Callable[ [types.Callable[..., types.Optional[types.Iterable[_T]]]], types.Callable[..., _TC], ]: - ''' + """ Convert any generator to a list or other type of collection. >>> @listify() @@ -97,10 +96,10 @@ def listify( >>> dict_generator() {'a': 1, 'b': 2} - ''' + """ def _listify( - function: types.Callable[..., types.Optional[types.Iterable[_T]]] + function: types.Callable[..., types.Optional[types.Iterable[_T]]], ) -> types.Callable[..., _TC]: def __listify(*args: types.Any, **kwargs: types.Any) -> _TC: result: types.Optional[types.Iterable[_T]] = function( @@ -123,7 +122,7 @@ def __listify(*args: types.Any, **kwargs: types.Any) -> _TC: def sample(sample_rate: float): - ''' + """ Limit calls to a function based on given sample rate. Number of calls to the function will be roughly equal to sample_rate percentage. @@ -135,10 +134,10 @@ def sample(sample_rate: float): ... return 1 Calls to *demo_function* will be limited to 50% approximatly. - ''' + """ def _sample( - function: types.Callable[_P, _T] + function: types.Callable[_P, _T], ) -> types.Callable[_P, types.Optional[_T]]: @functools.wraps(function) def __sample( @@ -152,7 +151,7 @@ def __sample( function, args, kwargs, - ) # noqa: E501 + ) return None return __sample @@ -168,10 +167,10 @@ def wraps_classmethod( ], types.Callable[types.Concatenate[types.Type[_S], _P], _T], ]: - ''' + """ Like `functools.wraps`, but for wrapping classmethods with the type info - from a regular method - ''' + from a regular method. + """ def _wraps_classmethod( wrapper: types.Callable[types.Concatenate[types.Any, _P], _T], diff --git a/python_utils/exceptions.py b/python_utils/exceptions.py index 3ffc01a..05cc0bd 100644 --- a/python_utils/exceptions.py +++ b/python_utils/exceptions.py @@ -6,7 +6,7 @@ def raise_exception( *args: types.Any, **kwargs: types.Any, ) -> types.Callable[..., None]: - ''' + """ Returns a function that raises an exception of the given type with the given arguments. @@ -14,7 +14,7 @@ def raise_exception( Traceback (most recent call last): ... ValueError: spam - ''' + """ def raise_(*args_: types.Any, **kwargs_: types.Any) -> types.Any: raise exception_class(*args, **kwargs) diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 04d661d..6c9f6cd 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -5,7 +5,7 @@ def camel_to_underscore(name: str) -> str: - '''Convert camel case style naming to underscore/snake case style naming + """Convert camel case style naming to underscore/snake case style naming. If there are existing underscores they will be collapsed with the to-be-added underscores. Multiple consecutive capital letters will not be @@ -21,7 +21,7 @@ def camel_to_underscore(name: str) -> str: '__spam_and_bacon__' >>> camel_to_underscore('__SpamANDBacon__') '__spam_and_bacon__' - ''' + """ output: types.List[str] = [] for i, c in enumerate(name): if i > 0: @@ -47,14 +47,19 @@ def apply_recursive( data: types.OptionalScope = None, **kwargs: types.Any, ) -> types.OptionalScope: - ''' - Apply a function to all keys in a scope recursively + """ + Apply a function to all keys in a scope recursively. >>> apply_recursive(camel_to_underscore, {'SpamEggsAndBacon': 'spam'}) {'spam_eggs_and_bacon': 'spam'} - >>> apply_recursive(camel_to_underscore, {'SpamEggsAndBacon': { - ... 'SpamEggsAndBacon': 'spam', - ... }}) + >>> apply_recursive( + ... camel_to_underscore, + ... { + ... 'SpamEggsAndBacon': { + ... 'SpamEggsAndBacon': 'spam', + ... } + ... }, + ... ) {'spam_eggs_and_bacon': {'spam_eggs_and_bacon': 'spam'}} >>> a = {'a_b_c': 123, 'def': {'DeF': 456}} @@ -63,7 +68,7 @@ def apply_recursive( {'a_b_c': 123, 'def': {'de_f': 456}} >>> apply_recursive(camel_to_underscore, None) - ''' + """ if data is None: return None @@ -80,7 +85,7 @@ def timesince( dt: types.Union[datetime.datetime, datetime.timedelta], default: str = 'just now', ) -> str: - ''' + """ Returns string representing 'time since' e.g. 3 days ago, 5 hours ago etc. @@ -121,7 +126,7 @@ def timesince( '1 hour and 2 minutes ago' >>> timesince(datetime.timedelta(seconds=3721)) '1 hour and 2 minutes ago' - ''' + """ if isinstance(dt, datetime.timedelta): diff = dt else: diff --git a/python_utils/generators.py b/python_utils/generators.py index 51b76b8..52bf7e4 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -4,7 +4,6 @@ import python_utils from python_utils import types - _T = types.TypeVar('_T') @@ -16,10 +15,10 @@ async def abatcher( batch_size: types.Optional[int] = None, interval: types.Optional[types.delta_type] = None, ) -> types.AsyncGenerator[types.List[_T], None]: - ''' + """ Asyncio generator wrapper that returns items with a given batch size or interval (whichever is reached first). - ''' + """ batch: types.List[_T] = [] assert batch_size or interval, 'Must specify either batch_size or interval' @@ -80,9 +79,7 @@ def batcher( iterable: types.Iterable[_T], batch_size: int = 10, ) -> types.Generator[types.List[_T], None, None]: - ''' - Generator wrapper that returns items with a given batch size - ''' + """Generator wrapper that returns items with a given batch size.""" batch: types.List[_T] = [] for item in iterable: batch.append(item) diff --git a/python_utils/import_.py b/python_utils/import_.py index b7008ae..14332ae 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -1,19 +1,23 @@ from . import types -class DummyException(Exception): +class DummyError(Exception): pass -def import_global( +# Legacy alias for DummyError +DummyException = DummyError + + +def import_global( # noqa: C901 name: str, modules: types.Optional[types.List[str]] = None, - exceptions: types.ExceptionsType = DummyException, + exceptions: types.ExceptionsType = DummyError, locals_: types.OptionalScope = None, globals_: types.OptionalScope = None, level: int = -1, -) -> types.Any: - '''Import the requested items into the global scope +) -> types.Any: # sourcery skip: hoist-if-from-if + """Import the requested items into the global scope. WARNING! this method _will_ overwrite your global scope If you have a variable named "path" and you call import_global('sys') @@ -22,12 +26,12 @@ def import_global( Args: name (str): the name of the module to import, e.g. sys modules (str): the modules to import, use None for everything - exception (Exception): the exception to catch, e.g. ImportError - `locals_`: the `locals()` method (in case you need a different scope) - `globals_`: the `globals()` method (in case you need a different scope) + exceptions (Exception): the exception to catch, e.g. ImportError + locals_: the `locals()` method (in case you need a different scope) + globals_: the `globals()` method (in case you need a different scope) level (int): the level to import from, this can be used for relative imports - ''' + """ frame = None name_parts: types.List[str] = name.split('.') modules_set: types.Set[str] = set() @@ -65,8 +69,10 @@ def import_global( try: for attr in name_parts[1:]: module = getattr(module, attr) - except AttributeError: - raise ImportError('No module named ' + '.'.join(name_parts)) + except AttributeError as e: + raise ImportError( + 'No module named ' + '.'.join(name_parts) + ) from e # If no list of modules is given, autodetect from either __all__ # or a dir() of the module diff --git a/python_utils/logger.py b/python_utils/logger.py index 9bd80cd..676b469 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -25,7 +25,7 @@ class LoggerBase(abc.ABC): - '''Class which automatically adds logging utilities to your class when + """Class which automatically adds logging utilities to your class when interiting. Expects `logger` to be a logging.Logger or compatible instance. Adds easy access to debug, info, warning, error, exception and log methods @@ -43,7 +43,7 @@ class LoggerBase(abc.ABC): >>> my_class.error('error') >>> my_class.exception('exception') >>> my_class.log(0, 'log') - ''' + """ # Being a tad lazy here and not creating a Protocol. # The actual classes define the correct type anyway @@ -199,8 +199,8 @@ def log( class Logged(LoggerBase): - '''Class which automatically adds a named logger to your class when - interiting + """Class which automatically adds a named logger to your class when + interiting. Adds easy access to debug, info, warning, error, exception and log methods @@ -218,7 +218,7 @@ class Logged(LoggerBase): >>> my_class._Logged__get_name('spam') 'spam' - ''' + """ logger: logging.Logger # pragma: no cover @@ -230,4 +230,4 @@ def __new__(cls, *args: types.Any, **kwargs: types.Any): cls.logger = logging.getLogger( cls.__get_name(cls.__module__, cls.__name__) ) - return super(Logged, cls).__new__(cls) + return super().__new__(cls) diff --git a/python_utils/loguru.py b/python_utils/loguru.py index 22b258d..24fd498 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -1,4 +1,5 @@ from __future__ import annotations + import typing import loguru diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 53948d8..43cd630 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -1,15 +1,17 @@ +from __future__ import annotations + import contextlib import os import typing from . import converters -Dimensions = typing.Tuple[int, int] +Dimensions = tuple[int, int] OptionalDimensions = typing.Optional[Dimensions] def get_terminal_size() -> Dimensions: # pragma: no cover - '''Get the current size of your terminal + """Get the current size of your terminal. Multiple returns are not always a good idea, but in this case it greatly simplifies the code so I believe it's justified. It's not the prettiest @@ -17,9 +19,9 @@ def get_terminal_size() -> Dimensions: # pragma: no cover Returns: width, height: Two integers containing width and height - ''' - w: typing.Optional[int] - h: typing.Optional[int] + """ + w: int | None + h: int | None with contextlib.suppress(Exception): # Default to 79 characters for IPython notebooks @@ -77,7 +79,7 @@ def get_terminal_size() -> Dimensions: # pragma: no cover def _get_terminal_size_windows() -> OptionalDimensions: # pragma: no cover res = None try: - from ctypes import windll, create_string_buffer # type: ignore + from ctypes import create_string_buffer, windll # type: ignore # stdin handle is -10 # stdout handle is -11 @@ -93,7 +95,7 @@ def _get_terminal_size_windows() -> OptionalDimensions: # pragma: no cover import struct (_, _, _, _, _, left, top, right, bottom, _, _) = struct.unpack( - "hhhhHhhhhhh", csbi.raw + 'hhhhHhhhhhh', csbi.raw ) w = right - left h = bottom - top @@ -123,17 +125,18 @@ def _get_terminal_size_tput() -> OptionalDimensions: # pragma: no cover ) output = proc.communicate(input=None) h = int(output[0]) - return w, h except Exception: return None + else: + return w, h def _get_terminal_size_linux() -> OptionalDimensions: # pragma: no cover - def ioctl_GWINSZ(fd): + def ioctl_gwinsz(fd): try: import fcntl - import termios import struct + import termios return struct.unpack( 'hh', @@ -142,12 +145,12 @@ def ioctl_GWINSZ(fd): except Exception: return None - size = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) + size = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2) if not size: with contextlib.suppress(Exception): fd = os.open(os.ctermid(), os.O_RDONLY) # type: ignore - size = ioctl_GWINSZ(fd) + size = ioctl_gwinsz(fd) os.close(fd) if not size: try: diff --git a/python_utils/time.py b/python_utils/time.py index 51cacc1..614c727 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -18,7 +18,7 @@ def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: - '''Convert a timedelta to seconds with the microseconds as fraction + """Convert a timedelta to seconds with the microseconds as fraction. Note that this method has become largely obsolete with the `timedelta.total_seconds()` method introduced in Python 2.7. @@ -32,7 +32,7 @@ def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: '1.000001' >>> '%.6f' % timedelta_to_seconds(timedelta(microseconds=1)) '0.000001' - ''' + """ # Only convert to float if needed if delta.microseconds: total = delta.microseconds * 1e-6 @@ -44,8 +44,8 @@ def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: def delta_to_seconds(interval: types.delta_type) -> float: - ''' - Convert a timedelta to seconds + """ + Convert a timedelta to seconds. >>> delta_to_seconds(datetime.timedelta(seconds=1)) 1 @@ -57,13 +57,13 @@ def delta_to_seconds(interval: types.delta_type) -> float: Traceback (most recent call last): ... TypeError: Unknown type ... - ''' + """ if isinstance(interval, datetime.timedelta): return timedelta_to_seconds(interval) elif isinstance(interval, (int, float)): return interval else: - raise TypeError('Unknown type %s: %r' % (type(interval), interval)) + raise TypeError(f'Unknown type {type(interval)}: {interval!r}') def delta_to_seconds_or_none( @@ -79,7 +79,7 @@ def format_time( timestamp: types.timestamp_type, precision: datetime.timedelta = datetime.timedelta(seconds=1), ) -> str: - '''Formats timedelta/datetime/seconds + """Formats timedelta/datetime/seconds. >>> format_time('1') '0:00:01' @@ -100,7 +100,7 @@ def format_time( ... TypeError: Unknown type ... - ''' + """ precision_seconds = precision.total_seconds() if isinstance(timestamp, str): @@ -138,7 +138,7 @@ def format_time( elif timestamp is None: return '--:--:--' else: - raise TypeError('Unknown type %s: %r' % (type(timestamp), timestamp)) + raise TypeError(f'Unknown type {type(timestamp)}: {timestamp!r}') def timeout_generator( @@ -150,10 +150,10 @@ def timeout_generator( interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, ): - ''' + """ Generator that walks through the given iterable (a counter by default) until the float_timeout is reached with a configurable float_interval - between items + between items. >>> for i in timeout_generator(0.1, 0.06): ... print(i) @@ -179,7 +179,7 @@ def timeout_generator( 0 1 2 - ''' + """ float_timeout: float = delta_to_seconds(timeout) float_interval: float = delta_to_seconds(interval) float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( @@ -207,7 +207,7 @@ def timeout_generator( async def aio_timeout_generator( - timeout: types.delta_type, + timeout: types.delta_type, # noqa: ASYNC109 interval: types.delta_type = datetime.timedelta(seconds=1), iterable: types.Union[ types.AsyncIterable[_T], types.Callable[..., types.AsyncIterable[_T]] @@ -215,10 +215,10 @@ async def aio_timeout_generator( interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, ) -> types.AsyncGenerator[_T, None]: - ''' + """ Async generator that walks through the given async iterable (a counter by default) until the float_timeout is reached with a configurable - float_interval between items + float_interval between items. The interval_exponent automatically increases the float_timeout with each run. Note that if the float_interval is less than 1, 1/interval_exponent @@ -228,7 +228,7 @@ async def aio_timeout_generator( Doctests and asyncio are not friends, so no examples. But this function is effectively the same as the `timeout_generator` but it uses `async for` instead. - ''' + """ float_timeout: float = delta_to_seconds(timeout) float_interval: float = delta_to_seconds(interval) float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( @@ -257,7 +257,7 @@ async def aio_timeout_generator( async def aio_generator_timeout_detector( generator: types.AsyncGenerator[_T, None], - timeout: types.Optional[types.delta_type] = None, + timeout: types.Optional[types.delta_type] = None, # noqa: ASYNC109 total_timeout: types.Optional[types.delta_type] = None, on_timeout: types.Optional[ types.Callable[ @@ -272,7 +272,7 @@ async def aio_generator_timeout_detector( ] = exceptions.reraise, **on_timeout_kwargs: types.Mapping[types.Text, types.Any], ) -> types.AsyncGenerator[_T, None]: - ''' + """ This function is used to detect if an asyncio generator has not yielded an element for a set amount of time. @@ -282,7 +282,7 @@ async def aio_generator_timeout_detector( If `on_timeout` is not specified, the exception is reraised. If `on_timeout` is `None`, the exception is silently ignored and the generator will finish as normal. - ''' + """ if total_timeout is None: total_timeout_end = None else: @@ -295,14 +295,16 @@ async def aio_generator_timeout_detector( while True: try: if total_timeout_end and time.perf_counter() >= total_timeout_end: - raise asyncio.TimeoutError('Total timeout reached') + raise asyncio.TimeoutError( # noqa: TRY301 + 'Total timeout reached' + ) # noqa: TRY301 if timeout_s: yield await asyncio.wait_for(generator.__anext__(), timeout_s) else: yield await generator.__anext__() - except asyncio.TimeoutError as exception: + except asyncio.TimeoutError as exception: # noqa: PERF203 if on_timeout is not None: await on_timeout( generator, @@ -333,16 +335,12 @@ def aio_generator_timeout_detector_decorator( ] = exceptions.reraise, **on_timeout_kwargs: types.Mapping[types.Text, types.Any], ): - ''' - A decorator wrapper for aio_generator_timeout_detector. - ''' + """A decorator wrapper for aio_generator_timeout_detector.""" def _timeout_detector_decorator( - generator: types.Callable[_P, types.AsyncGenerator[_T, None]] + generator: types.Callable[_P, types.AsyncGenerator[_T, None]], ) -> types.Callable[_P, types.AsyncGenerator[_T, None]]: - ''' - The decorator itself. - ''' + """The decorator itself.""" @functools.wraps(generator) def wrapper( diff --git a/python_utils/types.py b/python_utils/types.py index 01c319a..c4757f2 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,21 +1,25 @@ # pyright: reportWildcardImportFromLibrary=false import datetime import decimal -from typing_extensions import * # type: ignore # noqa: F403 -from typing import * # type: ignore # pragma: no cover # noqa: F403 +from re import Match, Pattern from types import * # type: ignore # pragma: no cover # noqa: F403 +from typing import * # type: ignore # pragma: no cover # noqa: F403 # import * does not import these in all Python versions -from typing import Pattern, BinaryIO, IO, TextIO, Match - # Quickhand for optional because it gets so much use. If only Python had # support for an optional type shorthand such as `SomeType?` instead of # `Optional[SomeType]`. -from typing import Optional as O # noqa - # Since the Union operator is only supported for Python 3.10, we'll create a # shorthand for it. -from typing import Union as U # noqa +from typing import ( + IO, + BinaryIO, + Optional as O, # noqa: N817 + TextIO, + Union as U, # noqa: N817 +) + +from typing_extensions import * # type: ignore # noqa: F403 Scope = Dict[str, Any] OptionalScope = O[Scope] diff --git a/setup.py b/setup.py index 306ace7..0e694ae 100644 --- a/setup.py +++ b/setup.py @@ -1,22 +1,25 @@ -import os -import typing + +import pathlib import setuptools +# pyright: reportUnknownMemberType=false + # To prevent importing about and thereby breaking the coverage info we use this # exec hack -about: typing.Dict[str, str] = {} +about: dict[str, str] = {} with open('python_utils/__about__.py') as fp: exec(fp.read(), about) -if os.path.isfile('README.rst'): - long_description = open('README.rst').read() +_readme_path = pathlib.Path(__file__).parent / 'README.rst' +if _readme_path.exists() and _readme_path.is_file(): + long_description = _readme_path.read_text() else: long_description = 'See http://pypi.python.org/pypi/python-utils/' if __name__ == '__main__': setuptools.setup( - python_requires='>3.8.0', + python_requires='>3.9.0', name='python-utils', version=about['__version__'], author=about['__author__'], @@ -30,7 +33,6 @@ package_data={'python_utils': ['py.typed']}, long_description=long_description, install_requires=['typing_extensions>3.10.0.2'], - tests_require=['pytest'], extras_require={ 'loguru': [ 'loguru', diff --git a/tox.ini b/tox.ini index f017232..288f7d1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = black, py38, py39, py310, py311, flake8, docs, mypy, pyright +envlist = ruff, black, py38, py39, py310, py311, flake8, docs, mypy, pyright skip_missing_interpreters = True [testenv] @@ -19,6 +19,11 @@ commands = pyright py.test --basetemp="{envtmpdir}" --confcutdir=.. {posargs} python_utils _python_utils_tests +[testenv:ruff] +basepython = python3 +deps = ruff +commands = ruff check {toxinidir}/setup.py {toxinidir}/_python_utils_tests {toxinidir}/python_utils + [testenv:black] basepython = python3 deps = black From af620d6e958e54f35e72dd82e8ffff729c86f3df Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sun, 22 Sep 2024 14:04:55 +0200 Subject: [PATCH 110/132] More docs --- python_utils/containers.py | 49 ++++++++++++++++++++++---------------- python_utils/time.py | 1 + python_utils/types.py | 11 +++++++++ setup.py | 8 +++++++ 4 files changed, 48 insertions(+), 21 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index 1809c7d..a004753 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -113,11 +113,11 @@ class CastedDictBase(types.Dict[KT, VT], abc.ABC): _value_cast: VT_cast[VT] def __init__( - self, - key_cast: KT_cast[KT] = None, - value_cast: VT_cast[VT] = None, - *args: DictUpdateArgs[KT, VT], - **kwargs: VT, + self, + key_cast: KT_cast[KT] = None, + value_cast: VT_cast[VT] = None, + *args: DictUpdateArgs[KT, VT], + **kwargs: VT, ) -> None: """ Initializes the CastedDictBase with optional key and value @@ -138,7 +138,9 @@ def __init__( self.update(*args, **kwargs) def update( - self, *args: DictUpdateArgs[types.Any, types.Any], **kwargs: types.Any + self, + *args: DictUpdateArgs[types.Any, types.Any], + **kwargs: types.Any ) -> None: if args: kwargs.update(*args) @@ -190,6 +192,7 @@ class CastedDict(CastedDictBase[KT, VT]): """ def __setitem__(self, key: typing.Any, value: typing.Any) -> None: + """Sets `key` to `cast(value)` in the dictionary.""" if self._value_cast is not None: value = self._value_cast(value) @@ -260,7 +263,7 @@ def __getitem__(self, key: types.Any) -> VT: return value def items( # type: ignore - self, + self, ) -> types.Generator[types.Tuple[KT, VT], None, None]: if self._value_cast is None: yield from super().items() @@ -313,9 +316,9 @@ class UniqueList(types.List[HT]): _set: types.Set[HT] def __init__( - self, - *args: HT, - on_duplicate: OnDuplicate = 'ignore', + self, + *args: HT, + on_duplicate: OnDuplicate = 'ignore', ): self.on_duplicate = on_duplicate self._set = set() @@ -348,18 +351,20 @@ def __contains__(self, item: HT) -> bool: # type: ignore @typing.overload def __setitem__( - self, indices: types.SupportsIndex, values: HT - ) -> None: ... + self, indices: types.SupportsIndex, values: HT + ) -> None: + ... @typing.overload def __setitem__( - self, indices: slice, values: types.Iterable[HT] - ) -> None: ... + self, indices: slice, values: types.Iterable[HT] + ) -> None: + ... def __setitem__( - self, - indices: types.Union[slice, types.SupportsIndex], - values: types.Union[types.Iterable[HT], HT], + self, + indices: types.Union[slice, types.SupportsIndex], + values: types.Union[types.Iterable[HT], HT], ) -> None: if isinstance(indices, slice): values = types.cast(types.Iterable[HT], values) @@ -389,7 +394,7 @@ def __setitem__( ) def __delitem__( - self, index: types.Union[types.SupportsIndex, slice] + self, index: types.Union[types.SupportsIndex, slice] ) -> None: if isinstance(index, slice): for value in self[index]: @@ -404,13 +409,15 @@ def __delitem__( # runtime, mypy and pyright currently so we have to ignore the errors class SliceableDeque(types.Generic[T], collections.deque): # type: ignore @typing.overload - def __getitem__(self, index: types.SupportsIndex) -> T: ... + def __getitem__(self, index: types.SupportsIndex) -> T: + ... @typing.overload - def __getitem__(self, index: slice) -> 'SliceableDeque[T]': ... + def __getitem__(self, index: slice) -> 'SliceableDeque[T]': + ... def __getitem__( - self, index: types.Union[types.SupportsIndex, slice] + self, index: types.Union[types.SupportsIndex, slice] ) -> types.Union[T, 'SliceableDeque[T]']: """ Return the item or slice at the given index. diff --git a/python_utils/time.py b/python_utils/time.py index 614c727..fca7048 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -69,6 +69,7 @@ def delta_to_seconds(interval: types.delta_type) -> float: def delta_to_seconds_or_none( interval: types.Optional[types.delta_type], ) -> types.Optional[float]: + """Convert a timedelta to seconds or return None.""" if interval is None: return None else: diff --git a/python_utils/types.py b/python_utils/types.py index c4757f2..d4f06c3 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -1,3 +1,14 @@ +""" +This module provides type definitions and utility functions for type hinting. + +It includes: +- Shorthand for commonly used types such as Optional and Union. +- Type aliases for various data structures and common types. +- Importing all types from the `typing` and `typing_extensions` modules. +- Importing specific types from the `types` module. + +The module also configures Pyright to ignore wildcard import warnings. +""" # pyright: reportWildcardImportFromLibrary=false import datetime import decimal diff --git a/setup.py b/setup.py index 0e694ae..07092ba 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,12 @@ +""" +Setup script for the python-utils package. +This script uses setuptools to package the python-utils library. It reads +metadata from the `python_utils/__about__.py` file and the `README.rst` file to +populate the package information. The script also defines the package +requirements and optional dependencies for different use cases such as logging, +documentation, and testing. +""" import pathlib import setuptools From 875e556fd8b380952afc1f0572c313d71b2c1093 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Mon, 23 Sep 2024 22:06:25 +0200 Subject: [PATCH 111/132] added ruff config --- ruff.toml | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 ruff.toml diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..1ce08a4 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,107 @@ +# We keep the ruff configuration separate so it can easily be shared across +# all projects + +target-version = 'py39' + +exclude = [ + '.venv', + '.tox', + 'test.py', +] + +lint.ignore = [ + 'A001', # Variable {name} is shadowing a Python builtin + 'A002', # Argument {name} is shadowing a Python builtin + 'A003', # Class attribute {name} is shadowing a Python builtin + 'B023', # function-uses-loop-variable + 'B024', # `FormatWidgetMixin` is an abstract base class, but it has no abstract methods + 'D205', # blank-line-after-summary + 'D212', # multi-line-summary-first-line + 'RET505', # Unnecessary `else` after `return` statement + 'TRY003', # Avoid specifying long messages outside the exception class + 'RET507', # Unnecessary `elif` after `continue` statement + 'C405', # Unnecessary {obj_type} literal (rewrite as a set literal) + 'C406', # Unnecessary {obj_type} literal (rewrite as a dict literal) + 'C408', # Unnecessary {obj_type} call (rewrite as a literal) + 'SIM114', # Combine `if` branches using logical `or` operator + 'RET506', # Unnecessary `else` after `raise` statement + 'Q001', # Remove bad quotes + 'Q002', # Remove bad quotes + 'COM812', # Missing trailing comma in a list + 'ISC001', # String concatenation with implicit str conversion + 'SIM108', # Ternary operators are not always more readable + 'RUF100', # Unused `noqa` directive. These vary per Python version so this warning is often incorrect. +] +line-length = 79 +lint.select = [ + 'A', # flake8-builtins + 'ASYNC', # flake8 async checker + 'B', # flake8-bugbear + 'C4', # flake8-comprehensions + 'C90', # mccabe + 'COM', # flake8-commas + + ## Require docstrings for all public methods, would be good to enable at some point + 'D', # pydocstyle + + 'E', # pycodestyle error ('W' for warning) + 'F', # pyflakes + 'FA', # flake8-future-annotations + 'I', # isort + 'ICN', # flake8-import-conventions + 'INP', # flake8-no-pep420 + 'ISC', # flake8-implicit-str-concat + 'N', # pep8-naming + 'NPY', # NumPy-specific rules + 'PERF', # perflint, + 'PIE', # flake8-pie + 'Q', # flake8-quotes + + 'RET', # flake8-return + 'RUF', # Ruff-specific rules + 'SIM', # flake8-simplify + 'T20', # flake8-print + 'TD', # flake8-todos + 'TRY', # tryceratops + 'UP', # pyupgrade +] + +[lint.per-file-ignores] +'*tests/*' = ['INP001', 'T201', 'T203', 'ASYNC109', 'B007'] +'examples.py' = ['T201', 'N806'] +'docs/conf.py' = ['E501', 'INP001'] +'docs/_theme/flask_theme_support.py' = ['RUF012', 'INP001'] +'*/types.py' = ['F405'] + +[lint.pydocstyle] +convention = 'google' +ignore-decorators = [ + 'typing.overload', + 'typing.override', +] + +[lint.isort] +case-sensitive = true +combine-as-imports = true +force-wrap-aliases = true + +[lint.flake8-quotes] +docstring-quotes = 'single' +inline-quotes = 'single' +multiline-quotes = 'single' + +[format] +line-ending = 'lf' +indent-style = 'space' +quote-style = 'single' +docstring-code-format = true +skip-magic-trailing-comma = false +exclude = [ + '__init__.py', +] + +[lint.pycodestyle] +max-line-length = 79 + +[lint.flake8-pytest-style] +mark-parentheses = true From c988ae317a1bc1b8d791263ef998ef6f289d4b77 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Sep 2024 01:44:49 +0200 Subject: [PATCH 112/132] added more documentation --- python_utils/containers.py | 227 +++++++++++++++++++++++++++++++------ python_utils/converters.py | 14 +++ python_utils/decorators.py | 15 +++ python_utils/exceptions.py | 17 +++ python_utils/formatters.py | 15 +++ python_utils/generators.py | 36 +++++- python_utils/import_.py | 19 +++- python_utils/logger.py | 38 +++++++ python_utils/loguru.py | 33 ++++++ python_utils/terminal.py | 14 +++ python_utils/time.py | 19 ++++ python_utils/types.py | 2 + setup.py | 1 + 13 files changed, 410 insertions(+), 40 deletions(-) diff --git a/python_utils/containers.py b/python_utils/containers.py index a004753..1d791cb 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -53,6 +53,7 @@ >>> d[1:4] SliceableDeque([2, 3, 4]) """ + # pyright: reportIncompatibleMethodOverride=false import abc import collections @@ -109,15 +110,16 @@ class CastedDictBase(types.Dict[KT, VT], abc.ABC): Sets the item in the dictionary, casting the key if a key cast callable is provided. """ + _key_cast: KT_cast[KT] _value_cast: VT_cast[VT] def __init__( - self, - key_cast: KT_cast[KT] = None, - value_cast: VT_cast[VT] = None, - *args: DictUpdateArgs[KT, VT], - **kwargs: VT, + self, + key_cast: KT_cast[KT] = None, + value_cast: VT_cast[VT] = None, + *args: DictUpdateArgs[KT, VT], + **kwargs: VT, ) -> None: """ Initializes the CastedDictBase with optional key and value @@ -138,10 +140,16 @@ def __init__( self.update(*args, **kwargs) def update( - self, - *args: DictUpdateArgs[types.Any, types.Any], - **kwargs: types.Any + self, *args: DictUpdateArgs[types.Any, types.Any], **kwargs: types.Any ) -> None: + """ + Updates the dictionary with the given arguments. + + Args: + *args (DictUpdateArgs[types.Any, types.Any]): Arguments to update + the dictionary. + **kwargs (types.Any): Keyword arguments to update the dictionary. + """ if args: kwargs.update(*args) @@ -150,6 +158,14 @@ def update( self[key] = value def __setitem__(self, key: types.Any, value: types.Any) -> None: + """ + Sets the item in the dictionary, casting the key if a key cast + callable is provided. + + Args: + key (types.Any): The key to set in the dictionary. + value (types.Any): The value to set in the dictionary. + """ if self._key_cast is not None: key = self._key_cast(key) @@ -246,12 +262,30 @@ class LazyCastedDict(CastedDictBase[KT, VT]): """ def __setitem__(self, key: types.Any, value: types.Any): + """ + Sets the item in the dictionary, casting the key if a key cast + callable is provided. + + Args: + key (types.Any): The key to set in the dictionary. + value (types.Any): The value to set in the dictionary. + """ if self._key_cast is not None: key = self._key_cast(key) super().__setitem__(key, value) def __getitem__(self, key: types.Any) -> VT: + """ + Gets the item from the dictionary, casting the value if a value cast + callable is provided. + + Args: + key (types.Any): The key to get from the dictionary. + + Returns: + VT: The value from the dictionary. + """ if self._key_cast is not None: key = self._key_cast(key) @@ -263,8 +297,16 @@ def __getitem__(self, key: types.Any) -> VT: return value def items( # type: ignore - self, + self, ) -> types.Generator[types.Tuple[KT, VT], None, None]: + """ + Returns a generator of the dictionary's items, casting the values if a + value cast callable is provided. + + Yields: + types.Generator[types.Tuple[KT, VT], None, None]: A generator of + the dictionary's items. + """ if self._value_cast is None: yield from super().items() else: @@ -272,6 +314,14 @@ def items( # type: ignore yield key, self._value_cast(value) def values(self) -> types.Generator[VT, None, None]: # type: ignore + """ + Returns a generator of the dictionary's values, casting the values if a + value cast callable is provided. + + Yields: + types.Generator[VT, None, None]: A generator of the dictionary's + values. + """ if self._value_cast is None: yield from super().values() else: @@ -316,10 +366,18 @@ class UniqueList(types.List[HT]): _set: types.Set[HT] def __init__( - self, - *args: HT, - on_duplicate: OnDuplicate = 'ignore', + self, + *args: HT, + on_duplicate: OnDuplicate = 'ignore', ): + """ + Initializes the UniqueList with optional duplicate handling behavior. + + Args: + *args (HT): Initial values for the list. + on_duplicate (OnDuplicate, optional): Behavior on duplicates. + Defaults to 'ignore'. + """ self.on_duplicate = on_duplicate self._set = set() super().__init__() @@ -327,6 +385,17 @@ def __init__( self.append(arg) def insert(self, index: types.SupportsIndex, value: HT) -> None: + """ + Inserts a value at the specified index, ensuring uniqueness. + + Args: + index (types.SupportsIndex): The index to insert the value at. + value (HT): The value to insert. + + Raises: + ValueError: If the value is a duplicate and `on_duplicate` is set + to 'raise'. + """ if value in self._set: if self.on_duplicate == 'raise': raise ValueError(f'Duplicate value: {value}') @@ -337,6 +406,16 @@ def insert(self, index: types.SupportsIndex, value: HT) -> None: super().insert(index, value) def append(self, value: HT) -> None: + """ + Appends a value to the list, ensuring uniqueness. + + Args: + value (HT): The value to append. + + Raises: + ValueError: If the value is a duplicate and `on_duplicate` is set + to 'raise'. + """ if value in self._set: if self.on_duplicate == 'raise': raise ValueError(f'Duplicate value: {value}') @@ -347,25 +426,45 @@ def append(self, value: HT) -> None: super().append(value) def __contains__(self, item: HT) -> bool: # type: ignore + """ + Checks if the list contains the specified item. + + Args: + item (HT): The item to check for. + + Returns: + bool: True if the item is in the list, False otherwise. + """ return item in self._set @typing.overload def __setitem__( - self, indices: types.SupportsIndex, values: HT - ) -> None: - ... + self, indices: types.SupportsIndex, values: HT + ) -> None: ... @typing.overload def __setitem__( - self, indices: slice, values: types.Iterable[HT] - ) -> None: - ... + self, indices: slice, values: types.Iterable[HT] + ) -> None: ... def __setitem__( - self, - indices: types.Union[slice, types.SupportsIndex], - values: types.Union[types.Iterable[HT], HT], + self, + indices: types.Union[slice, types.SupportsIndex], + values: types.Union[types.Iterable[HT], HT], ) -> None: + """ + Sets the item(s) at the specified index/indices, ensuring uniqueness. + + Args: + indices (types.Union[slice, types.SupportsIndex]): The index or + slice to set the value(s) at. + values (types.Union[types.Iterable[HT], HT]): The value(s) to set. + + Raises: + RuntimeError: If `on_duplicate` is 'ignore' and setting slices. + ValueError: If the value(s) are duplicates and `on_duplicate` is + set to 'raise'. + """ if isinstance(indices, slice): values = types.cast(types.Iterable[HT], values) if self.on_duplicate == 'ignore': @@ -394,8 +493,15 @@ def __setitem__( ) def __delitem__( - self, index: types.Union[types.SupportsIndex, slice] + self, index: types.Union[types.SupportsIndex, slice] ) -> None: + """ + Deletes the item(s) at the specified index/indices. + + Args: + index (types.Union[types.SupportsIndex, slice]): The index or slice + to delete the item(s) at. + """ if isinstance(index, slice): for value in self[index]: self._set.remove(value) @@ -408,28 +514,49 @@ def __delitem__( # Type hinting `collections.deque` does not work consistently between Python # runtime, mypy and pyright currently so we have to ignore the errors class SliceableDeque(types.Generic[T], collections.deque): # type: ignore + """ + A deque that supports slicing and enhanced equality checks. + + Methods: + __getitem__(index: types.Union[types.SupportsIndex, slice]) -> + types.Union[T, 'SliceableDeque[T]']: + Returns the item or slice at the given index. + __eq__(other: types.Any) -> bool: + Checks equality with another object, allowing for comparison with + lists, tuples, and sets. + pop(index: int = -1) -> T: + Removes and returns the item at the given index. Only supports + index 0 and the last index. + """ + @typing.overload - def __getitem__(self, index: types.SupportsIndex) -> T: - ... + def __getitem__(self, index: types.SupportsIndex) -> T: ... @typing.overload - def __getitem__(self, index: slice) -> 'SliceableDeque[T]': - ... + def __getitem__(self, index: slice) -> 'SliceableDeque[T]': ... def __getitem__( - self, index: types.Union[types.SupportsIndex, slice] + self, index: types.Union[types.SupportsIndex, slice] ) -> types.Union[T, 'SliceableDeque[T]']: """ Return the item or slice at the given index. - >>> d = SliceableDeque[int]([1, 2, 3, 4, 5]) - >>> d[1:4] - SliceableDeque([2, 3, 4]) + Args: + index (types.Union[types.SupportsIndex, slice]): The index or + slice to retrieve. - >>> d = SliceableDeque[str](['a', 'b', 'c']) - >>> d[-2:] - SliceableDeque(['b', 'c']) + Returns: + types.Union[T, 'SliceableDeque[T]']: The item or slice at the + given index. + Examples: + >>> d = SliceableDeque[int]([1, 2, 3, 4, 5]) + >>> d[1:4] + SliceableDeque([2, 3, 4]) + + >>> d = SliceableDeque[str](['a', 'b', 'c']) + >>> d[-2:] + SliceableDeque(['b', 'c']) """ if isinstance(index, slice): start, stop, step = index.indices(len(self)) @@ -438,7 +565,16 @@ def __getitem__( return types.cast(T, super().__getitem__(index)) def __eq__(self, other: types.Any) -> bool: - # Allow for comparison with a list or tuple + """ + Checks equality with another object, allowing for comparison with + lists, tuples, and sets. + + Args: + other (types.Any): The object to compare with. + + Returns: + bool: True if the objects are equal, False otherwise. + """ if isinstance(other, list): return list(self) == other elif isinstance(other, tuple): @@ -449,8 +585,27 @@ def __eq__(self, other: types.Any) -> bool: return super().__eq__(other) def pop(self, index: int = -1) -> T: - # We need to allow for an index but a deque only allows the removal of - # the first or last item. + """ + Removes and returns the item at the given index. Only supports index 0 + and the last index. + + Args: + index (int, optional): The index of the item to remove. Defaults to + -1. + + Returns: + T: The removed item. + + Raises: + IndexError: If the index is not 0 or the last index. + + Examples: + >>> d = SliceableDeque([1, 2, 3]) + >>> d.pop(0) + 1 + >>> d.pop() + 3 + """ if index == 0: return typing.cast(T, super().popleft()) elif index in {-1, len(self) - 1}: diff --git a/python_utils/converters.py b/python_utils/converters.py index da221c5..69e51f8 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -1,3 +1,17 @@ +""" +This module provides utility functions for type conversion. + +Functions: + - to_int: Convert a string to an integer with optional regular expression + matching. + - to_float: Convert a string to a float with optional regular expression + matching. + - to_unicode: Convert objects to Unicode strings. + - to_str: Convert objects to byte strings. + - scale_1024: Scale a number down to a suitable size based on powers of + 1024. + - remap: Remap a value from one range to another. +""" from __future__ import annotations import decimal diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 0459f88..9780fb5 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -1,3 +1,18 @@ +""" +This module provides various utility decorators for Python functions +and methods. + +The decorators include: + +1. `set_attributes`: Sets attributes on functions and classes. +2. `listify`: Converts any generator to a list or other collection. +3. `sample`: Limits calls to a function based on a sample rate. +4. `wraps_classmethod`: Wraps classmethods with type info from a + regular method. + +Each decorator is designed to enhance the functionality of Python +functions and methods in a simple and reusable manner. +""" import contextlib import functools import logging diff --git a/python_utils/exceptions.py b/python_utils/exceptions.py index 05cc0bd..fe6750e 100644 --- a/python_utils/exceptions.py +++ b/python_utils/exceptions.py @@ -1,3 +1,14 @@ +""" +This module provides utility functions for raising and reraising exceptions. + +Functions: + raise_exception(exception_class, *args, **kwargs): + Returns a function that raises an exception of the given type with + the given arguments. + + reraise(*args, **kwargs): + Reraises the current exception. +""" from . import types @@ -23,4 +34,10 @@ def raise_(*args_: types.Any, **kwargs_: types.Any) -> types.Any: def reraise(*args: types.Any, **kwargs: types.Any) -> types.Any: + """ + Reraises the current exception. + + This function seems useless, but it can be useful when you need to pass + a callable to another function that raises an exception. + """ raise diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 6c9f6cd..ac1d03d 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -1,3 +1,18 @@ +""" +This module provides utility functions for formatting strings and dates. + +Functions: + camel_to_underscore(name: str) -> str: + Convert camel case style naming to underscore/snake case style naming. + + apply_recursive(function: Callable[[str], str], data: OptionalScope = None, + **kwargs: Any) -> OptionalScope: + Apply a function to all keys in a scope recursively. + + timesince(dt: Union[datetime.datetime, datetime.timedelta], + default: str = 'just now') -> str: + Returns string representing 'time since' e.g. 3 days ago, 5 hours ago. +""" # pyright: reportUnnecessaryIsInstance=false import datetime diff --git a/python_utils/generators.py b/python_utils/generators.py index 52bf7e4..8ff06af 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -1,3 +1,15 @@ +""" +This module provides generator utilities for batching items from +iterables and async iterables. + +Functions: + abatcher(generator, batch_size=None, interval=None): + Asyncio generator wrapper that returns items with a given batch + size or interval (whichever is reached first). + + batcher(iterable, batch_size=10): + Generator wrapper that returns items with a given batch size. +""" import asyncio import time @@ -18,6 +30,17 @@ async def abatcher( """ Asyncio generator wrapper that returns items with a given batch size or interval (whichever is reached first). + + Args: + generator: The async generator or iterator to batch. + batch_size (types.Optional[int], optional): The number of items per + batch. Defaults to None. + interval (types.Optional[types.delta_type], optional): The time + interval to wait before yielding a batch. Defaults to None. + + Yields: + types.AsyncGenerator[types.List[_T], None]: A generator that yields + batches of items. """ batch: types.List[_T] = [] @@ -79,7 +102,18 @@ def batcher( iterable: types.Iterable[_T], batch_size: int = 10, ) -> types.Generator[types.List[_T], None, None]: - """Generator wrapper that returns items with a given batch size.""" + """ + Generator wrapper that returns items with a given batch size. + + Args: + iterable (types.Iterable[_T]): The iterable to batch. + batch_size (int, optional): The number of items per batch. Defaults + to 10. + + Yields: + types.Generator[types.List[_T], None, None]: A generator that yields + batches of items. + """ batch: types.List[_T] = [] for item in iterable: batch.append(item) diff --git a/python_utils/import_.py b/python_utils/import_.py index 14332ae..f1201a8 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -1,8 +1,21 @@ +""" +This module provides utilities for importing modules and handling exceptions. + +Classes: + DummyError(Exception): + A custom exception class used as a default for exception handling. + +Functions: + import_global(name, modules=None, exceptions=DummyError, locals_=None, + globals_=None, level=-1): + Imports the requested items into the global scope, with support for + relative imports and custom exception handling. +""" from . import types class DummyError(Exception): - pass + """A custom exception class used as a default for exception handling.""" # Legacy alias for DummyError @@ -20,8 +33,8 @@ def import_global( # noqa: C901 """Import the requested items into the global scope. WARNING! this method _will_ overwrite your global scope - If you have a variable named "path" and you call import_global('sys') - it will be overwritten with sys.path + If you have a variable named `path` and you call `import_global('sys')` + it will be overwritten with `sys.path` Args: name (str): the name of the module to import, e.g. sys diff --git a/python_utils/logger.py b/python_utils/logger.py index 676b469..5ae6bb0 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -1,3 +1,29 @@ +""" +This module provides a base class `LoggerBase` and a derived class `Logged` +for adding logging capabilities to classes. The `LoggerBase` class expects +a `logger` attribute to be a `logging.Logger` or compatible instance and +provides methods for logging at various levels. The `Logged` class +automatically adds a named logger to the class. + +Classes: + LoggerBase: + A base class that adds logging utilities to a class. + Logged: + A derived class that automatically adds a named logger to a class. + +Example: + >>> class MyClass(Logged): + ... def __init__(self): + ... Logged.__init__(self) + + >>> my_class = MyClass() + >>> my_class.debug('debug') + >>> my_class.info('info') + >>> my_class.warning('warning') + >>> my_class.error('error') + >>> my_class.exception('exception') + >>> my_class.log(0, 'log') +""" import abc import logging @@ -227,6 +253,18 @@ def __get_name(cls, *name_parts: str) -> str: return LoggerBase._LoggerBase__get_name(*name_parts) # type: ignore def __new__(cls, *args: types.Any, **kwargs: types.Any): + """ + Create a new instance of the class and initialize the logger. + + The logger is named using the module and class name. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + An instance of the class. + """ cls.logger = logging.getLogger( cls.__get_name(cls.__module__, cls.__name__) ) diff --git a/python_utils/loguru.py b/python_utils/loguru.py index 24fd498..1771c61 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -1,3 +1,18 @@ +""" +This module provides a `Logurud` class that integrates the `loguru` logger +with the base logging functionality defined in `logger_module.LoggerBase`. + +Classes: + Logurud: A class that extends `LoggerBase` and uses `loguru` for logging. + +Usage example: + >>> from python_utils.loguru import Logurud + >>> class MyClass(Logurud): + ... def __init__(self): + ... Logurud.__init__(self) + >>> my_class = MyClass() + >>> my_class.logger.info('This is an info message') +""" from __future__ import annotations import typing @@ -10,8 +25,26 @@ class Logurud(logger_module.LoggerBase): + """ + A class that extends `LoggerBase` and uses `loguru` for logging. + + Attributes: + logger (loguru.Logger): The `loguru` logger instance. + """ + logger: loguru.Logger def __new__(cls, *args: typing.Any, **kwargs: typing.Any): + """ + Creates a new instance of `Logurud` and initializes the `loguru` + logger. + + Args: + *args (typing.Any): Variable length argument list. + **kwargs (typing.Any): Arbitrary keyword arguments. + + Returns: + Logurud: A new instance of `Logurud`. + """ cls.logger: loguru.Logger = loguru.logger.opt(depth=1) return super().__new__(cls) diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 43cd630..5222464 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -1,3 +1,17 @@ +""" +This module provides functions to get the terminal size across different +platforms. + +Functions: + get_terminal_size: Get the current size of the terminal. + _get_terminal_size_windows: Get terminal size on Windows. + _get_terminal_size_tput: Get terminal size using `tput`. + _get_terminal_size_linux: Get terminal size on Linux. + +Usage example: + >>> width, height = get_terminal_size() + >>> print(f"Width: {width}, Height: {height}") +""" from __future__ import annotations import contextlib diff --git a/python_utils/time.py b/python_utils/time.py index fca7048..f86e001 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -1,3 +1,22 @@ +""" +This module provides utility functions for handling time-related operations. + +Functions: +- timedelta_to_seconds: Convert a timedelta to seconds with microseconds as + fraction. +- delta_to_seconds: Convert a timedelta or numeric interval to seconds. +- delta_to_seconds_or_none: Convert a timedelta to seconds or return None. +- format_time: Format a timestamp (timedelta, datetime, or seconds) to a + string. +- timeout_generator: Generate items from an iterable until a timeout is + reached. +- aio_timeout_generator: Asynchronously generate items from an iterable until a + timeout is reached. +- aio_generator_timeout_detector: Detect if an async generator has not yielded + an element for a set amount of time. +- aio_generator_timeout_detector_decorator: Decorator for + aio_generator_timeout_detector. +""" # pyright: reportUnnecessaryIsInstance=false import asyncio import datetime diff --git a/python_utils/types.py b/python_utils/types.py index d4f06c3..d9b5138 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -10,6 +10,8 @@ The module also configures Pyright to ignore wildcard import warnings. """ # pyright: reportWildcardImportFromLibrary=false +# ruff: noqa: F405 + import datetime import decimal from re import Match, Pattern diff --git a/setup.py b/setup.py index 07092ba..9de14fa 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,7 @@ 'sphinx', 'types-setuptools', 'loguru', + 'loguru-mypy', ], }, classifiers=['License :: OSI Approved :: BSD License'], From 74cbe8639875f639550be12d0f7cbc00bee7fe45 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Sep 2024 01:56:28 +0200 Subject: [PATCH 113/132] Modernized python versions --- .github/workflows/main.yml | 2 +- tox.ini | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2f8ac44..834086d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 4 strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v3 diff --git a/tox.ini b/tox.ini index 288f7d1..5fae06b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,13 +1,14 @@ [tox] -envlist = ruff, black, py38, py39, py310, py311, flake8, docs, mypy, pyright +envlist = ruff, black, py39, py310, py311, py312, py313, flake8, docs, mypy, pyright skip_missing_interpreters = True [testenv] basepython = - py38: python3.8 py39: python3.9 py310: python3.10 py311: python3.11 + py312: python3.12 + py313: python3.13 setenv = PY_IGNORE_IMPORTMISMATCH=1 deps = From 1763037b0c53e8f8bafb97af8d7652f827135979 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Sep 2024 01:57:32 +0200 Subject: [PATCH 114/132] Removed obsolete compat module --- python_utils/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python_utils/__init__.py b/python_utils/__init__.py index e9fd9b1..9a6c629 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -4,7 +4,6 @@ Submodules: aio - compat converters decorators formatters @@ -52,7 +51,6 @@ from . import ( aio, - compat, converters, decorators, formatters, @@ -87,7 +85,6 @@ __all__ = [ 'aio', 'generators', - 'compat', 'converters', 'decorators', 'formatters', From 2c1a22d9c6468abb461efe63781fad6871032080 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Sep 2024 01:58:52 +0200 Subject: [PATCH 115/132] Disabled python 3.13, Github actions does not support it yet --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 834086d..f64db40 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 4 strategy: matrix: - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.9', '3.10', '3.11', '3.12'] # Maybe soon?, '3.13'] steps: - uses: actions/checkout@v3 From 5dbe56c3a31a25a33af809189d19fd0d3a615bc1 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Sep 2024 02:03:29 +0200 Subject: [PATCH 116/132] =?UTF-8?q?ruff=20does=20everything=20that=20flake?= =?UTF-8?q?8=20does=20and=20better,=20so=20let's=20replace=20it=EF=A3=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/main.yml | 12 ++++++------ CONTRIBUTING.md | 10 ++++++++-- setup.py | 3 ++- tox.ini | 7 +------ 4 files changed, 17 insertions(+), 15 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f64db40..327ffd1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,14 +21,14 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - python -m pip install --upgrade pip setuptools flake8 + python -m pip install --upgrade pip setuptools ruff pip install -e '.[tests]' - name: Get versions run: | python -V pip freeze - - name: flake8 - run: flake8 -v python_utils setup.py + - name: ruff + run: ruff check - name: pytest run: py.test @@ -44,12 +44,12 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools - pip install -e '.[docs,tests]' pyright flake8 mypy + pip install -e '.[docs,tests]' pyright ruff mypy - name: build docs run: make html working-directory: docs/ - - name: flake8 - run: flake8 -v python_utils setup.py + - name: ruff + run: ruff check - name: mypy run: mypy python_utils setup.py - name: pyright diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b2e961e..d13f7da 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,8 +74,14 @@ Run tests $ py.test ``` -Note that this won't run `flake8` yet, so once all the tests succeed you can run `flake8` to check for code style errors. +Note that this won't run `ruff` yet, so once all the tests succeed you can run `ruff check` to check for code style errors. ```bash -$ flake8 +$ ruff check +``` + +Lastly we test the types using `pyright`: + +```bash +$ pyright ``` diff --git a/setup.py b/setup.py index 9de14fa..31c482d 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,8 @@ 'python-utils', ], 'tests': [ - 'flake8', + 'ruff', + 'pyright', 'pytest', 'pytest-cov', 'pytest-mypy', diff --git a/tox.ini b/tox.ini index 5fae06b..1c8148e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = ruff, black, py39, py310, py311, py312, py313, flake8, docs, mypy, pyright +envlist = ruff, black, py39, py310, py311, py312, py313, docs, mypy, pyright skip_missing_interpreters = True [testenv] @@ -30,11 +30,6 @@ basepython = python3 deps = black commands = black --skip-string-normalization --line-length 79 {toxinidir}/setup.py {toxinidir}/_python_utils_tests {toxinidir}/python_utils -[testenv:flake8] -basepython = python3 -deps = flake8 -commands = flake8 python_utils {posargs} - [testenv:pyright] basepython = python3 deps = From 7ef8b4252b2cf24b001e2c50d19988b17519ac81 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 24 Sep 2024 02:10:35 +0200 Subject: [PATCH 117/132] Fixed test coverage --- .github/workflows/main.yml | 4 ++-- _python_utils_tests/test_aio.py | 7 +++++++ python_utils/aio.py | 2 +- python_utils/terminal.py | 1 - 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 327ffd1..ea0db00 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,7 +28,7 @@ jobs: python -V pip freeze - name: ruff - run: ruff check + run: ruff check --output-format=github - name: pytest run: py.test @@ -49,7 +49,7 @@ jobs: run: make html working-directory: docs/ - name: ruff - run: ruff check + run: ruff check --output-format=github - name: mypy run: mypy python_utils setup.py - name: pyright diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index db3ffa5..a4d624a 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -58,4 +58,11 @@ async def async_gen(): yield 3, 4 yield 5, 6 + async def empty_gen(): + if False: + yield 1, 2 + assert await adict(async_gen) == {1: 2, 3: 4, 5: 6} + assert await adict(async_gen()) == {1: 2, 3: 4, 5: 6} + assert await adict(empty_gen) == {} + assert await adict(empty_gen()) == {} diff --git a/python_utils/aio.py b/python_utils/aio.py index 0146c54..80eec6c 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -109,7 +109,7 @@ async def adict( item: types.Tuple[_K, _V] items: types.List[types.Tuple[_K, _V]] = [] - async for item in iterable_: + async for item in iterable_: # pragma: no branch items.append(item) return container(items) diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 5222464..95eb26b 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -10,7 +10,6 @@ Usage example: >>> width, height = get_terminal_size() - >>> print(f"Width: {width}, Height: {height}") """ from __future__ import annotations From 8fc4ffa6d85b381c8b23891e59cbfa8a627d80f3 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 25 Sep 2024 03:42:50 +0200 Subject: [PATCH 118/132] thoroughly fixed all typing issues for both mypy and pyright and removed all type ignores --- _python_utils_tests/test_aio.py | 16 +-- _python_utils_tests/test_decorators.py | 31 +++--- _python_utils_tests/test_generators.py | 8 +- _python_utils_tests/test_import.py | 14 +-- _python_utils_tests/test_logger.py | 3 +- _python_utils_tests/test_python_utils.py | 2 +- _python_utils_tests/test_time.py | 28 ++--- pyproject.toml | 6 ++ python_utils/aio.py | 5 +- python_utils/containers.py | 16 +-- python_utils/converters.py | 129 +++++++++++++++++------ python_utils/decorators.py | 20 ++-- python_utils/logger.py | 100 ++++++++++++++++-- python_utils/loguru.py | 2 +- python_utils/terminal.py | 23 ++-- python_utils/time.py | 77 +++++++++----- python_utils/types.py | 6 +- setup.py | 2 + 18 files changed, 338 insertions(+), 150 deletions(-) diff --git a/_python_utils_tests/test_aio.py b/_python_utils_tests/test_aio.py index a4d624a..9096f10 100644 --- a/_python_utils_tests/test_aio.py +++ b/_python_utils_tests/test_aio.py @@ -7,10 +7,10 @@ @pytest.mark.asyncio -async def test_acount(monkeypatch: pytest.MonkeyPatch): +async def test_acount(monkeypatch: pytest.MonkeyPatch) -> None: sleeps: types.List[float] = [] - async def mock_sleep(delay: float): + async def mock_sleep(delay: float) -> None: sleeps.append(delay) monkeypatch.setattr(asyncio, 'sleep', mock_sleep) @@ -23,13 +23,13 @@ async def mock_sleep(delay: float): @pytest.mark.asyncio -async def test_acontainer(): - async def async_gen(): +async def test_acontainer() -> None: + async def async_gen() -> types.AsyncIterable[int]: yield 1 yield 2 yield 3 - async def empty_gen(): + async def empty_gen() -> types.AsyncIterable[int]: if False: yield 1 @@ -52,13 +52,13 @@ async def empty_gen(): @pytest.mark.asyncio -async def test_adict(): - async def async_gen(): +async def test_adict() -> None: + async def async_gen() -> types.AsyncIterable[types.Tuple[int, int]]: yield 1, 2 yield 3, 4 yield 5, 6 - async def empty_gen(): + async def empty_gen() -> types.AsyncIterable[types.Tuple[int, int]]: if False: yield 1, 2 diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py index 16d40e0..3f339e4 100644 --- a/_python_utils_tests/test_decorators.py +++ b/_python_utils_tests/test_decorators.py @@ -1,9 +1,12 @@ +import typing from unittest.mock import MagicMock import pytest from python_utils.decorators import sample, wraps_classmethod +T = typing.TypeVar('T') + @pytest.fixture def random(monkeypatch: pytest.MonkeyPatch) -> MagicMock: @@ -14,7 +17,7 @@ def random(monkeypatch: pytest.MonkeyPatch) -> MagicMock: return mock -def test_sample_called(random: MagicMock): +def test_sample_called(random: MagicMock) -> None: demo_function = MagicMock() decorated = sample(0.5)(demo_function) random.return_value = 0.4 @@ -28,7 +31,7 @@ def test_sample_called(random: MagicMock): assert demo_function.call_count == 3 -def test_sample_not_called(random: MagicMock): +def test_sample_not_called(random: MagicMock) -> None: demo_function = MagicMock() decorated = sample(0.5)(demo_function) random.return_value = 0.5 @@ -40,31 +43,31 @@ def test_sample_not_called(random: MagicMock): class SomeClass: @classmethod - def some_classmethod(cls, arg): # type: ignore - return arg # type: ignore + def some_classmethod(cls, arg: T) -> T: + return arg @classmethod def some_annotated_classmethod(cls, arg: int) -> int: return arg -def test_wraps_classmethod(): # type: ignore +def test_wraps_classmethod() -> None: some_class = SomeClass() - some_class.some_classmethod = MagicMock() - wrapped_method = wraps_classmethod( # type: ignore - SomeClass.some_classmethod # type: ignore - )( # type: ignore - some_class.some_classmethod # type: ignore + some_class.some_classmethod = MagicMock() # type: ignore[method-assign] + wrapped_method = wraps_classmethod( + SomeClass.some_classmethod + )( + some_class.some_classmethod ) wrapped_method(123) - some_class.some_classmethod.assert_called_with(123) # type: ignore + some_class.some_classmethod.assert_called_with(123) -def test_wraps_annotated_classmethod(): # type: ignore +def test_wraps_annotated_classmethod() -> None: some_class = SomeClass() - some_class.some_annotated_classmethod = MagicMock() + some_class.some_annotated_classmethod = MagicMock() # type: ignore[method-assign] wrapped_method = wraps_classmethod(SomeClass.some_annotated_classmethod)( some_class.some_annotated_classmethod ) - wrapped_method(123) # type: ignore + wrapped_method(123) some_class.some_annotated_classmethod.assert_called_with(123) diff --git a/_python_utils_tests/test_generators.py b/_python_utils_tests/test_generators.py index 9a80795..39498e0 100644 --- a/_python_utils_tests/test_generators.py +++ b/_python_utils_tests/test_generators.py @@ -7,7 +7,7 @@ @pytest.mark.asyncio -async def test_abatcher(): +async def test_abatcher() -> None: async for batch in python_utils.abatcher(python_utils.acount(stop=9), 3): assert len(batch) == 3 @@ -28,8 +28,8 @@ async def test_abatcher_timed() -> None: @pytest.mark.asyncio -async def test_abatcher_timed_with_timeout(): - async def generator(): +async def test_abatcher_timed_with_timeout() -> None: + async def generator() -> types.AsyncIterator[int]: # Test if the timeout is respected yield 0 yield 1 @@ -57,7 +57,7 @@ async def generator(): await batcher.__anext__() -def test_batcher(): +def test_batcher() -> None: batch = [] for batch in python_utils.batcher(range(9), 3): assert len(batch) == 3 diff --git a/_python_utils_tests/test_import.py b/_python_utils_tests/test_import.py index 9de85ba..31be2be 100644 --- a/_python_utils_tests/test_import.py +++ b/_python_utils_tests/test_import.py @@ -1,19 +1,19 @@ from python_utils import import_, types -def test_import_globals_relative_import(): +def test_import_globals_relative_import() -> None: for i in range(-1, 5): relative_import(i) -def relative_import(level: int): +def relative_import(level: int) -> None: locals_: types.Dict[str, types.Any] = {} globals_ = {'__name__': 'python_utils.import_'} import_.import_global('.formatters', locals_=locals_, globals_=globals_) assert 'camel_to_underscore' in globals_ -def test_import_globals_without_inspection(): +def test_import_globals_without_inspection() -> None: locals_: types.Dict[str, types.Any] = {} globals_: types.Dict[str, types.Any] = {'__name__': __name__} import_.import_global( @@ -22,7 +22,7 @@ def test_import_globals_without_inspection(): assert 'camel_to_underscore' in globals_ -def test_import_globals_single_method(): +def test_import_globals_single_method() -> None: locals_: types.Dict[str, types.Any] = {} globals_: types.Dict[str, types.Any] = {'__name__': __name__} import_.import_global( @@ -34,19 +34,19 @@ def test_import_globals_single_method(): assert 'camel_to_underscore' in globals_ -def test_import_globals_with_inspection(): +def test_import_globals_with_inspection() -> None: import_.import_global('python_utils.formatters') assert 'camel_to_underscore' in globals() -def test_import_globals_missing_module(): +def test_import_globals_missing_module() -> None: import_.import_global( 'python_utils.spam', exceptions=ImportError, locals_=locals() ) assert 'camel_to_underscore' in globals() -def test_import_locals_missing_module(): +def test_import_locals_missing_module() -> None: import_.import_global( 'python_utils.spam', exceptions=ImportError, globals_=globals() ) diff --git a/_python_utils_tests/test_logger.py b/_python_utils_tests/test_logger.py index a443603..2d26696 100644 --- a/_python_utils_tests/test_logger.py +++ b/_python_utils_tests/test_logger.py @@ -1,3 +1,4 @@ +# mypy: disable-error-code=misc import pytest from python_utils.loguru import Logurud @@ -5,7 +6,7 @@ loguru = pytest.importorskip('loguru') -def test_logurud(): +def test_logurud() -> None: class MyClass(Logurud): pass diff --git a/_python_utils_tests/test_python_utils.py b/_python_utils_tests/test_python_utils.py index 0ced509..5a41d4b 100644 --- a/_python_utils_tests/test_python_utils.py +++ b/_python_utils_tests/test_python_utils.py @@ -1,7 +1,7 @@ from python_utils import __about__ -def test_definitions(): +def test_definitions() -> None: # The setup.py requires this so we better make sure they exist :) assert __about__.__version__ assert __about__.__author__ diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index ffd1de5..e29c516 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -32,7 +32,7 @@ async def test_aio_timeout_generator( maximum_interval: float, iterable: types.AsyncIterable[types.Any], result: int, -): +) -> None: i = None async for i in python_utils.aio_timeout_generator( timeout, interval, iterable, maximum_interval=maximum_interval @@ -46,7 +46,7 @@ async def test_aio_timeout_generator( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ (0.01, 0.006, 0.5, 0.01, 'abc', 'c'), - (0.01, 0.006, 0.5, 0.01, itertools.count, 2), # type: ignore + (0.01, 0.006, 0.5, 0.01, itertools.count, 2), (0.01, 0.006, 0.5, 0.01, itertools.count(), 2), (0.01, 0.006, 1.0, None, 'abc', 'c'), ( @@ -54,7 +54,7 @@ async def test_aio_timeout_generator( timedelta(seconds=0.006), 2.0, timedelta(seconds=0.01), - itertools.count, # type: ignore + itertools.count, 2, ), ], @@ -70,7 +70,7 @@ def test_timeout_generator( types.Callable[..., types.Iterable[types.Any]], ], result: int, -): +) -> None: i = None for i in python_utils.timeout_generator( timeout=timeout, @@ -85,11 +85,11 @@ def test_timeout_generator( @pytest.mark.asyncio -async def test_aio_generator_timeout_detector(): +async def test_aio_generator_timeout_detector() -> None: # Make pyright happy i = None - async def generator(): + async def generator() -> types.AsyncGenerator[int, None]: for i in range(10): await asyncio.sleep(i / 100.0) yield i @@ -123,10 +123,10 @@ async def generator(): @pytest.mark.asyncio -async def test_aio_generator_timeout_detector_decorator_reraise(): +async def test_aio_generator_timeout_detector_decorator_reraise() -> None: # Test regular timeout with reraise @python_utils.aio_generator_timeout_detector_decorator(timeout=0.05) - async def generator_timeout(): + async def generator_timeout() -> types.AsyncGenerator[int, None]: for i in range(10): await asyncio.sleep(i / 100.0) yield i @@ -137,7 +137,7 @@ async def generator_timeout(): @pytest.mark.asyncio -async def test_aio_generator_timeout_detector_decorator_clean_exit(): +async def test_aio_generator_timeout_detector_decorator_clean_exit() -> None: # Make pyright happy i = None @@ -145,7 +145,7 @@ async def test_aio_generator_timeout_detector_decorator_clean_exit(): @python_utils.aio_generator_timeout_detector_decorator( timeout=0.05, on_timeout=None ) - async def generator_clean(): + async def generator_clean() -> types.AsyncGenerator[int, None]: for i in range(10): await asyncio.sleep(i / 100.0) yield i @@ -157,10 +157,10 @@ async def generator_clean(): @pytest.mark.asyncio -async def test_aio_generator_timeout_detector_decorator_reraise_total(): +async def test_aio_generator_timeout_detector_decorator_reraise_total() -> None: # Test total timeout with reraise @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.1) - async def generator_reraise(): + async def generator_reraise() -> types.AsyncGenerator[int, None]: for i in range(10): await asyncio.sleep(i / 100.0) yield i @@ -171,7 +171,7 @@ async def generator_reraise(): @pytest.mark.asyncio -async def test_aio_generator_timeout_detector_decorator_clean_total(): +async def test_aio_generator_timeout_detector_decorator_clean_total() -> None: # Make pyright happy i = None @@ -179,7 +179,7 @@ async def test_aio_generator_timeout_detector_decorator_clean_total(): @python_utils.aio_generator_timeout_detector_decorator( total_timeout=0.1, on_timeout=None ) - async def generator_clean_total(): + async def generator_clean_total() -> types.AsyncGenerator[int, None]: for i in range(10): await asyncio.sleep(i / 100.0) yield i diff --git a/pyproject.toml b/pyproject.toml index 4f8224e..f1400f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,3 +11,9 @@ strict = ['python_utils', '_python_utils_tests', 'setup.py'] ignore = ['python_utils/terminal.py'] pythonVersion = '3.9' +[tool.mypy] +strict = true +check_untyped_defs = true + +[[tool.mypy.overrides]] +module = '_python_utils_tests.*' diff --git a/python_utils/aio.py b/python_utils/aio.py index 80eec6c..9df2150 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -62,8 +62,9 @@ async def acontainer( types.AsyncIterable[_T], types.Callable[..., types.AsyncIterable[_T]], ], - container: types.Callable[[types.Iterable[_T]], types.Iterable[_T]] = list, -) -> types.Iterable[_T]: + container: types.Callable[[types.Iterable[_T]], types.Collection[_T]] = + list, +) -> types.Collection[_T]: """ Asyncio version of list()/set()/tuple()/etc() using an async for loop. diff --git a/python_utils/containers.py b/python_utils/containers.py index 1d791cb..e7e7c4a 100644 --- a/python_utils/containers.py +++ b/python_utils/containers.py @@ -261,7 +261,7 @@ class LazyCastedDict(CastedDictBase[KT, VT]): '4' """ - def __setitem__(self, key: types.Any, value: types.Any): + def __setitem__(self, key: types.Any, value: types.Any) -> None: """ Sets the item in the dictionary, casting the key if a key cast callable is provided. @@ -296,7 +296,7 @@ def __getitem__(self, key: types.Any) -> VT: return value - def items( # type: ignore + def items( # type: ignore[override] self, ) -> types.Generator[types.Tuple[KT, VT], None, None]: """ @@ -313,7 +313,7 @@ def items( # type: ignore for key, value in super().items(): yield key, self._value_cast(value) - def values(self) -> types.Generator[VT, None, None]: # type: ignore + def values(self) -> types.Generator[VT, None, None]: # type: ignore[override] """ Returns a generator of the dictionary's values, casting the values if a value cast callable is provided. @@ -425,7 +425,7 @@ def append(self, value: HT) -> None: self._set.add(value) super().append(value) - def __contains__(self, item: HT) -> bool: # type: ignore + def __contains__(self, item: HT) -> bool: # type: ignore[override] """ Checks if the list contains the specified item. @@ -513,7 +513,7 @@ def __delitem__( # Type hinting `collections.deque` does not work consistently between Python # runtime, mypy and pyright currently so we have to ignore the errors -class SliceableDeque(types.Generic[T], collections.deque): # type: ignore +class SliceableDeque(types.Generic[T], collections.deque[T]): """ A deque that supports slicing and enhanced equality checks. @@ -562,7 +562,7 @@ def __getitem__( start, stop, step = index.indices(len(self)) return self.__class__(self[i] for i in range(start, stop, step)) else: - return types.cast(T, super().__getitem__(index)) + return super().__getitem__(index) def __eq__(self, other: types.Any) -> bool: """ @@ -607,9 +607,9 @@ def pop(self, index: int = -1) -> T: 3 """ if index == 0: - return typing.cast(T, super().popleft()) + return super().popleft() elif index in {-1, len(self) - 1}: - return typing.cast(T, super().pop()) + return super().pop() else: raise IndexError( 'Only index 0 and the last index (`N-1` or `-1`) ' diff --git a/python_utils/converters.py b/python_utils/converters.py index 69e51f8..cbe949f 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -12,11 +12,14 @@ 1024. - remap: Remap a value from one range to another. """ +# Ignoring all mypy errors because mypy doesn't understand many modern typing +# constructs... please, use pyright instead if you can. from __future__ import annotations import decimal import math import re +import typing from typing import Union from . import types @@ -24,7 +27,8 @@ _TN = types.TypeVar('_TN', bound=types.DecimalNumber) _RegexpType: types.TypeAlias = Union[ - types.Pattern[str], str, types.Literal[True], None] + types.Pattern[str], str, types.Literal[True], None +] def to_int( @@ -111,7 +115,7 @@ def to_int( return default else: return int(input_) - except exception: # type: ignore + except exception: return default @@ -131,7 +135,7 @@ def to_float( in a string. When True it will automatically match any digit in the string. When a (regexp) object (has a search method) is given, that will be used. - WHen a string is given, re.compile will be run over it first + When a string is given, re.compile will be run over it first The last group of the regexp will be used as value @@ -278,7 +282,69 @@ def scale_1024( return scaled, power +@typing.overload +def remap( + value: decimal.Decimal, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal | float, +) -> decimal.Decimal: ... + + +@typing.overload +def remap( + value: decimal.Decimal| float, + old_min: decimal.Decimal, + old_max: decimal.Decimal| float, + new_min: decimal.Decimal| float, + new_max: decimal.Decimal| float, +) -> decimal.Decimal: ... + + +@typing.overload +def remap( + value: decimal.Decimal | float, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal | float, +) -> decimal.Decimal: ... + + +@typing.overload +def remap( + value: decimal.Decimal | float, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal, + new_max: decimal.Decimal | float, +) -> decimal.Decimal: ... + + +@typing.overload +def remap( + value: decimal.Decimal | float, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal, +) -> decimal.Decimal: ... + + +# Note that float captures both int and float types so we don't need to +# specify them separately +@typing.overload def remap( + value: float, + old_min: float, + old_max: float, + new_min: float, + new_max: float, +) -> float: ... + + +def remap( # pyright: ignore[reportInconsistentOverload] value: _TN, old_min: _TN, old_max: _TN, @@ -330,32 +396,25 @@ def remap( ... ValueError: Output range (0-0) is empty - :param value: value to be converted - :type value: int, float, decimal.Decimal - - :param old_min: minimum of the range for the value that has been passed - :type old_min: int, float, decimal.Decimal - - :param old_max: maximum of the range for the value that has been passed - :type old_max: int, float, decimal.Decimal - - :param new_min: the minimum of the new range - :type new_min: int, float, decimal.Decimal - - :param new_max: the maximum of the new range - :type new_max: int, float, decimal.Decimal - - :return: value that has been re ranged. if any of the parameters passed is - a `decimal.Decimal` all of the parameters will be converted to - `decimal.Decimal`. The same thing also happens if one of the - parameters is a `float`. otherwise all parameters will get converted - into an `int`. technically you can pass a `str` of an integer and it - will get converted. The returned value type will be `decimal.Decimal` - of any of the passed parameters ar `decimal.Decimal`, the return type - will be `float` if any of the passed parameters are a `float` otherwise - the returned type will be `int`. - - :rtype: int, float, decimal.Decimal + Args: + value (int, float, decimal.Decimal): Value to be converted. + old_min (int, float, decimal.Decimal): Minimum of the range for the + value that has been passed. + old_max (int, float, decimal.Decimal): Maximum of the range for the + value that has been passed. + new_min (int, float, decimal.Decimal): The minimum of the new range. + new_max (int, float, decimal.Decimal): The maximum of the new range. + + Returns: int, float, decimal.Decimal: Value that has been re-ranged. If + any of the parameters passed is a `decimal.Decimal`, all of the + parameters will be converted to `decimal.Decimal`. The same thing also + happens if one of the parameters is a `float`. Otherwise, all + parameters will get converted into an `int`. Technically, you can pass + a `str` of an integer and it will get converted. The returned value + type will be `decimal.Decimal` if any of the passed parameters are + `decimal.Decimal`, the return type will be `float` if any of the + passed parameters are a `float`, otherwise the returned type will be + `int`. """ type_: types.Type[types.DecimalNumber] if ( @@ -374,7 +433,6 @@ def remap( or isinstance(new_max, float) ): type_ = float - else: type_ = int @@ -395,13 +453,16 @@ def remap( if new_range == 0: raise ValueError(f'Output range ({new_min}-{new_max}) is empty') - new_value = (value - old_min) * new_range # type: ignore + # The current state of Python typing makes it impossible to use the + # generic type system in this case. Or so extremely verbose that it's not + # worth it. + new_value = (value - old_min) * new_range # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] if type_ is int: - new_value //= old_range # type: ignore + new_value //= old_range # pyright: ignore[reportUnknownVariableType] else: - new_value /= old_range # type: ignore + new_value /= old_range # pyright: ignore[reportUnknownVariableType] - new_value += new_min # type: ignore + new_value += new_min # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] return types.cast(_TN, new_value) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index 9780fb5..d4f414a 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -21,7 +21,6 @@ from . import types _T = types.TypeVar('_T') -_TC = types.TypeVar('_TC', bound=types.Container[types.Any]) _P = types.ParamSpec('_P') _S = types.TypeVar('_S', covariant=True) @@ -61,11 +60,12 @@ def _set_attributes( def listify( - collection: types.Callable[[types.Iterable[_T]], _TC] = list, # type: ignore + collection: types.Callable[[types.Iterable[_T]], types.Collection[_T]] = + list, allow_empty: bool = True, ) -> types.Callable[ [types.Callable[..., types.Optional[types.Iterable[_T]]]], - types.Callable[..., _TC], + types.Callable[..., types.Collection[_T]], ]: """ Convert any generator to a list or other type of collection. @@ -115,8 +115,9 @@ def listify( def _listify( function: types.Callable[..., types.Optional[types.Iterable[_T]]], - ) -> types.Callable[..., _TC]: - def __listify(*args: types.Any, **kwargs: types.Any) -> _TC: + ) -> types.Callable[..., types.Collection[_T]]: + def __listify(*args: types.Any, **kwargs: types.Any) \ + -> types.Collection[_T]: result: types.Optional[types.Iterable[_T]] = function( *args, **kwargs ) @@ -136,7 +137,10 @@ def __listify(*args: types.Any, **kwargs: types.Any) -> _TC: return _listify -def sample(sample_rate: float): +def sample(sample_rate: float) -> types.Callable[ + [types.Callable[_P, _T]], + types.Callable[_P, types.Optional[_T]], +]: """ Limit calls to a function based on given sample rate. Number of calls to the function will be roughly equal to @@ -180,7 +184,7 @@ def wraps_classmethod( [ types.Callable[types.Concatenate[types.Any, _P], _T], ], - types.Callable[types.Concatenate[types.Type[_S], _P], _T], + types.Callable[types.Concatenate[_S, _P], _T], ]: """ Like `functools.wraps`, but for wrapping classmethods with the type info @@ -189,7 +193,7 @@ def wraps_classmethod( def _wraps_classmethod( wrapper: types.Callable[types.Concatenate[types.Any, _P], _T], - ) -> types.Callable[types.Concatenate[types.Type[_S], _P], _T]: + ) -> types.Callable[types.Concatenate[_S, _P], _T]: # For some reason `functools.update_wrapper` fails on some test # runs but not while running actual code with contextlib.suppress(AttributeError): diff --git a/python_utils/logger.py b/python_utils/logger.py index 5ae6bb0..d63d0df 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -50,6 +50,79 @@ _T = types.TypeVar('_T', covariant=True) +class LoggerProtocol(types.Protocol): + def debug( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: ... + + def info( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: ... + + def warning( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: ... + + def error( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: ... + + def critical( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: ... + + def exception( + self, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: ... + + def log( + self, + level: int, + msg: object, + *args: object, + exc_info: _ExcInfoType = None, + stack_info: bool = False, + stacklevel: int = 1, + extra: types.Union[types.Mapping[str, object], None] = None, + ) -> None: ... + + class LoggerBase(abc.ABC): """Class which automatically adds logging utilities to your class when interiting. Expects `logger` to be a logging.Logger or compatible instance. @@ -71,9 +144,11 @@ class LoggerBase(abc.ABC): >>> my_class.log(0, 'log') """ - # Being a tad lazy here and not creating a Protocol. - # The actual classes define the correct type anyway + # I've tried using a protocol to properly type the logger but it gave all + # sorts of issues with mypy so we're using the lazy solution for now. The + # actual classes define the correct type anyway logger: types.Any + # logger: LoggerProtocol @classmethod def __get_name( # pyright: ignore[reportUnusedFunction] @@ -92,7 +167,7 @@ def debug( stacklevel: int = 1, extra: types.Union[types.Mapping[str, object], None] = None, ) -> None: - return cls.logger.debug( + return cls.logger.debug( # type: ignore[no-any-return] msg, *args, exc_info=exc_info, @@ -112,7 +187,7 @@ def info( stacklevel: int = 1, extra: types.Union[types.Mapping[str, object], None] = None, ) -> None: - return cls.logger.info( + return cls.logger.info( # type: ignore[no-any-return] msg, *args, exc_info=exc_info, @@ -132,7 +207,7 @@ def warning( stacklevel: int = 1, extra: types.Union[types.Mapping[str, object], None] = None, ) -> None: - return cls.logger.warning( + return cls.logger.warning( # type: ignore[no-any-return] msg, *args, exc_info=exc_info, @@ -152,7 +227,7 @@ def error( stacklevel: int = 1, extra: types.Union[types.Mapping[str, object], None] = None, ) -> None: - return cls.logger.error( + return cls.logger.error( # type: ignore[no-any-return] msg, *args, exc_info=exc_info, @@ -172,7 +247,7 @@ def critical( stacklevel: int = 1, extra: types.Union[types.Mapping[str, object], None] = None, ) -> None: - return cls.logger.critical( + return cls.logger.critical( # type: ignore[no-any-return] msg, *args, exc_info=exc_info, @@ -192,7 +267,7 @@ def exception( stacklevel: int = 1, extra: types.Union[types.Mapping[str, object], None] = None, ) -> None: - return cls.logger.exception( + return cls.logger.exception( # type: ignore[no-any-return] msg, *args, exc_info=exc_info, @@ -213,7 +288,7 @@ def log( stacklevel: int = 1, extra: types.Union[types.Mapping[str, object], None] = None, ) -> None: - return cls.logger.log( + return cls.logger.log( # type: ignore[no-any-return] level, msg, *args, @@ -250,9 +325,12 @@ class Logged(LoggerBase): @classmethod def __get_name(cls, *name_parts: str) -> str: - return LoggerBase._LoggerBase__get_name(*name_parts) # type: ignore + return types.cast( + str, + LoggerBase._LoggerBase__get_name(*name_parts) # type: ignore[attr-defined] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue] + ) - def __new__(cls, *args: types.Any, **kwargs: types.Any): + def __new__(cls, *args: types.Any, **kwargs: types.Any) -> 'Logged': """ Create a new instance of the class and initialize the logger. diff --git a/python_utils/loguru.py b/python_utils/loguru.py index 1771c61..9352ffd 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -34,7 +34,7 @@ class Logurud(logger_module.LoggerBase): logger: loguru.Logger - def __new__(cls, *args: typing.Any, **kwargs: typing.Any): + def __new__(cls, *args: typing.Any, **kwargs: typing.Any) -> Logurud: """ Creates a new instance of `Logurud` and initializes the `loguru` logger. diff --git a/python_utils/terminal.py b/python_utils/terminal.py index 95eb26b..af611cd 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -21,6 +21,8 @@ Dimensions = tuple[int, int] OptionalDimensions = typing.Optional[Dimensions] +_StrDimensions = tuple[str, str] +_OptionalStrDimensions = typing.Optional[_StrDimensions] def get_terminal_size() -> Dimensions: # pragma: no cover @@ -38,10 +40,10 @@ def get_terminal_size() -> Dimensions: # pragma: no cover with contextlib.suppress(Exception): # Default to 79 characters for IPython notebooks - from IPython import get_ipython # type: ignore + from IPython import get_ipython # type: ignore[attr-defined] - ipython = get_ipython() - from ipykernel import zmqshell # type: ignore + ipython = get_ipython() # type: ignore[no-untyped-call] + from ipykernel import zmqshell # type: ignore[import-not-found] if isinstance(ipython, zmqshell.ZMQInteractiveShell): return 79, 24 @@ -61,7 +63,7 @@ def get_terminal_size() -> Dimensions: # pragma: no cover if w and h: return w, h with contextlib.suppress(Exception): - import blessings # type: ignore + import blessings # type: ignore[import-untyped] terminal = blessings.Terminal() w = terminal.width @@ -92,7 +94,7 @@ def get_terminal_size() -> Dimensions: # pragma: no cover def _get_terminal_size_windows() -> OptionalDimensions: # pragma: no cover res = None try: - from ctypes import create_string_buffer, windll # type: ignore + from ctypes import create_string_buffer, windll # type: ignore[attr-defined] # stdin handle is -10 # stdout handle is -11 @@ -145,24 +147,25 @@ def _get_terminal_size_tput() -> OptionalDimensions: # pragma: no cover def _get_terminal_size_linux() -> OptionalDimensions: # pragma: no cover - def ioctl_gwinsz(fd): + def ioctl_gwinsz(fd: int) -> tuple[str, str] | None: try: import fcntl import struct import termios - return struct.unpack( + return typing.cast(_OptionalStrDimensions, struct.unpack( 'hh', - fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'), # type: ignore - ) + fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'), # type: ignore[call-overload] + )) except Exception: return None + size: _OptionalStrDimensions size = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2) if not size: with contextlib.suppress(Exception): - fd = os.open(os.ctermid(), os.O_RDONLY) # type: ignore + fd = os.open(os.ctermid(), os.O_RDONLY) size = ioctl_gwinsz(fd) os.close(fd) if not size: diff --git a/python_utils/time.py b/python_utils/time.py index f86e001..224d8e1 100644 --- a/python_utils/time.py +++ b/python_utils/time.py @@ -17,6 +17,7 @@ - aio_generator_timeout_detector_decorator: Decorator for aio_generator_timeout_detector. """ + # pyright: reportUnnecessaryIsInstance=false import asyncio import datetime @@ -62,7 +63,7 @@ def timedelta_to_seconds(delta: datetime.timedelta) -> types.Number: return total -def delta_to_seconds(interval: types.delta_type) -> float: +def delta_to_seconds(interval: types.delta_type) -> types.Number: """ Convert a timedelta to seconds. @@ -87,7 +88,7 @@ def delta_to_seconds(interval: types.delta_type) -> float: def delta_to_seconds_or_none( interval: types.Optional[types.delta_type], -) -> types.Optional[float]: +) -> types.Optional[types.Number]: """Convert a timedelta to seconds or return None.""" if interval is None: return None @@ -161,21 +162,57 @@ def format_time( raise TypeError(f'Unknown type {type(timestamp)}: {timestamp!r}') +@types.overload +def _to_iterable( + iterable: types.Union[ + types.Callable[[], types.AsyncIterable[_T]], + types.AsyncIterable[_T], + ], +) -> types.AsyncIterable[_T]: ... + + +@types.overload +def _to_iterable( + iterable: types.Union[ + types.Callable[[], types.Iterable[_T]], types.Iterable[_T] + ], +) -> types.Iterable[_T]: ... + + +def _to_iterable( + iterable: types.Union[ + types.Iterable[_T], + types.Callable[[], types.Iterable[_T]], + types.AsyncIterable[_T], + types.Callable[[], types.AsyncIterable[_T]], + ], +) -> types.Union[types.Iterable[_T], types.AsyncIterable[_T]]: + if callable(iterable): + return iterable() + else: + return iterable + + def timeout_generator( timeout: types.delta_type, interval: types.delta_type = datetime.timedelta(seconds=1), iterable: types.Union[ types.Iterable[_T], types.Callable[[], types.Iterable[_T]] - ] = itertools.count, # type: ignore + ] = itertools.count, # type: ignore[assignment] interval_multiplier: float = 1.0, maximum_interval: types.Optional[types.delta_type] = None, -): +) -> types.Iterable[_T]: """ Generator that walks through the given iterable (a counter by default) until the float_timeout is reached with a configurable float_interval between items. + This can be used to limit the time spent on a slow operation. This can be + useful for testing slow APIs so you get a small sample of the data in a + reasonable amount of time. + >>> for i in timeout_generator(0.1, 0.06): + ... # Put your slow code here ... print(i) 0 1 @@ -200,19 +237,13 @@ def timeout_generator( 1 2 """ - float_timeout: float = delta_to_seconds(timeout) float_interval: float = delta_to_seconds(interval) float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( maximum_interval ) + iterable_ = _to_iterable(iterable) - iterable_: types.Iterable[_T] - if callable(iterable): - iterable_ = iterable() - else: - iterable_ = iterable - - end = float_timeout + time.perf_counter() + end = delta_to_seconds(timeout) + time.perf_counter() for item in iterable_: yield item @@ -221,7 +252,7 @@ def timeout_generator( time.sleep(float_interval) - interval *= interval_multiplier + float_interval *= interval_multiplier if float_maximum_interval: float_interval = min(float_interval, float_maximum_interval) @@ -249,19 +280,13 @@ async def aio_timeout_generator( effectively the same as the `timeout_generator` but it uses `async for` instead. """ - float_timeout: float = delta_to_seconds(timeout) float_interval: float = delta_to_seconds(interval) float_maximum_interval: types.Optional[float] = delta_to_seconds_or_none( maximum_interval ) + iterable_ = _to_iterable(iterable) - iterable_: types.AsyncIterable[_T] - if callable(iterable): - iterable_ = iterable() - else: - iterable_ = iterable - - end = float_timeout + time.perf_counter() + end = delta_to_seconds(timeout) + time.perf_counter() async for item in iterable_: # pragma: no branch yield item @@ -317,7 +342,7 @@ async def aio_generator_timeout_detector( if total_timeout_end and time.perf_counter() >= total_timeout_end: raise asyncio.TimeoutError( # noqa: TRY301 'Total timeout reached' - ) # noqa: TRY301 + ) if timeout_s: yield await asyncio.wait_for(generator.__anext__(), timeout_s) @@ -354,7 +379,10 @@ def aio_generator_timeout_detector_decorator( ] ] = exceptions.reraise, **on_timeout_kwargs: types.Mapping[types.Text, types.Any], -): +) -> types.Callable[ + [types.Callable[_P, types.AsyncGenerator[_T, None]]], + types.Callable[_P, types.AsyncGenerator[_T, None]], +]: """A decorator wrapper for aio_generator_timeout_detector.""" def _timeout_detector_decorator( @@ -364,7 +392,8 @@ def _timeout_detector_decorator( @functools.wraps(generator) def wrapper( - *args: _P.args, **kwargs: _P.kwargs + *args: _P.args, + **kwargs: _P.kwargs, ) -> types.AsyncGenerator[_T, None]: return aio_generator_timeout_detector( generator(*args, **kwargs), diff --git a/python_utils/types.py b/python_utils/types.py index d9b5138..90bcd2a 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -15,8 +15,8 @@ import datetime import decimal from re import Match, Pattern -from types import * # type: ignore # pragma: no cover # noqa: F403 -from typing import * # type: ignore # pragma: no cover # noqa: F403 +from types import * # pragma: no cover # noqa: F403 +from typing import * # pragma: no cover # noqa: F403 # import * does not import these in all Python versions # Quickhand for optional because it gets so much use. If only Python had @@ -32,7 +32,7 @@ Union as U, # noqa: N817 ) -from typing_extensions import * # type: ignore # noqa: F403 +from typing_extensions import * # type: ignore[no-redef,assignment] # noqa: F403 Scope = Dict[str, Any] OptionalScope = O[Scope] diff --git a/setup.py b/setup.py index 31c482d..ba77b56 100644 --- a/setup.py +++ b/setup.py @@ -61,6 +61,8 @@ 'types-setuptools', 'loguru', 'loguru-mypy', + 'mypy-ipython', + 'blessings', ], }, classifiers=['License :: OSI Approved :: BSD License'], From 3940aac6bc11c5d676db0d0137bc0806f4f35308 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 25 Sep 2024 03:46:13 +0200 Subject: [PATCH 119/132] ruff formatting --- _python_utils_tests/test_decorators.py | 4 +-- _python_utils_tests/test_time.py | 4 ++- docs/conf.py | 1 + python_utils/aio.py | 5 +-- python_utils/converters.py | 47 +++++++++++++------------- python_utils/decorators.py | 15 +++++--- python_utils/exceptions.py | 1 + python_utils/formatters.py | 1 + python_utils/generators.py | 1 + python_utils/import_.py | 1 + python_utils/logger.py | 3 +- python_utils/loguru.py | 1 + python_utils/terminal.py | 17 +++++++--- setup.py | 1 + 14 files changed, 62 insertions(+), 40 deletions(-) diff --git a/_python_utils_tests/test_decorators.py b/_python_utils_tests/test_decorators.py index 3f339e4..281698b 100644 --- a/_python_utils_tests/test_decorators.py +++ b/_python_utils_tests/test_decorators.py @@ -54,9 +54,7 @@ def some_annotated_classmethod(cls, arg: int) -> int: def test_wraps_classmethod() -> None: some_class = SomeClass() some_class.some_classmethod = MagicMock() # type: ignore[method-assign] - wrapped_method = wraps_classmethod( - SomeClass.some_classmethod - )( + wrapped_method = wraps_classmethod(SomeClass.some_classmethod)( some_class.some_classmethod ) wrapped_method(123) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index e29c516..a0f5c77 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -157,7 +157,9 @@ async def generator_clean() -> types.AsyncGenerator[int, None]: @pytest.mark.asyncio -async def test_aio_generator_timeout_detector_decorator_reraise_total() -> None: +async def test_aio_generator_timeout_detector_decorator_reraise_total() -> ( + None +): # Test total timeout with reraise @python_utils.aio_generator_timeout_detector_decorator(total_timeout=0.1) async def generator_reraise() -> types.AsyncGenerator[int, None]: diff --git a/docs/conf.py b/docs/conf.py index 1fcd968..44fac03 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ documentation root, use os.path.abspath to make it absolute, like shown here. # """ + import os import sys from datetime import date diff --git a/python_utils/aio.py b/python_utils/aio.py index 9df2150..7a7b3b3 100644 --- a/python_utils/aio.py +++ b/python_utils/aio.py @@ -62,8 +62,9 @@ async def acontainer( types.AsyncIterable[_T], types.Callable[..., types.AsyncIterable[_T]], ], - container: types.Callable[[types.Iterable[_T]], types.Collection[_T]] = - list, + container: types.Callable[ + [types.Iterable[_T]], types.Collection[_T] + ] = list, ) -> types.Collection[_T]: """ Asyncio version of list()/set()/tuple()/etc() using an async for loop. diff --git a/python_utils/converters.py b/python_utils/converters.py index cbe949f..c4240f3 100644 --- a/python_utils/converters.py +++ b/python_utils/converters.py @@ -12,6 +12,7 @@ 1024. - remap: Remap a value from one range to another. """ + # Ignoring all mypy errors because mypy doesn't understand many modern typing # constructs... please, use pyright instead if you can. from __future__ import annotations @@ -284,41 +285,41 @@ def scale_1024( @typing.overload def remap( - value: decimal.Decimal, - old_min: decimal.Decimal | float, - old_max: decimal.Decimal | float, - new_min: decimal.Decimal | float, - new_max: decimal.Decimal | float, + value: decimal.Decimal, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal | float, ) -> decimal.Decimal: ... @typing.overload def remap( - value: decimal.Decimal| float, - old_min: decimal.Decimal, - old_max: decimal.Decimal| float, - new_min: decimal.Decimal| float, - new_max: decimal.Decimal| float, + value: decimal.Decimal | float, + old_min: decimal.Decimal, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal | float, ) -> decimal.Decimal: ... @typing.overload def remap( - value: decimal.Decimal | float, - old_min: decimal.Decimal | float, - old_max: decimal.Decimal, - new_min: decimal.Decimal | float, - new_max: decimal.Decimal | float, + value: decimal.Decimal | float, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal, + new_min: decimal.Decimal | float, + new_max: decimal.Decimal | float, ) -> decimal.Decimal: ... @typing.overload def remap( - value: decimal.Decimal | float, - old_min: decimal.Decimal | float, - old_max: decimal.Decimal | float, - new_min: decimal.Decimal, - new_max: decimal.Decimal | float, + value: decimal.Decimal | float, + old_min: decimal.Decimal | float, + old_max: decimal.Decimal | float, + new_min: decimal.Decimal, + new_max: decimal.Decimal | float, ) -> decimal.Decimal: ... @@ -459,10 +460,10 @@ def remap( # pyright: ignore[reportInconsistentOverload] new_value = (value - old_min) * new_range # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] if type_ is int: - new_value //= old_range # pyright: ignore[reportUnknownVariableType] + new_value //= old_range # pyright: ignore[reportUnknownVariableType] else: - new_value /= old_range # pyright: ignore[reportUnknownVariableType] + new_value /= old_range # pyright: ignore[reportUnknownVariableType] - new_value += new_min # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] + new_value += new_min # type: ignore[operator] # pyright: ignore[reportOperatorIssue, reportUnknownVariableType] return types.cast(_TN, new_value) diff --git a/python_utils/decorators.py b/python_utils/decorators.py index d4f414a..8799581 100644 --- a/python_utils/decorators.py +++ b/python_utils/decorators.py @@ -13,6 +13,7 @@ Each decorator is designed to enhance the functionality of Python functions and methods in a simple and reusable manner. """ + import contextlib import functools import logging @@ -60,8 +61,9 @@ def _set_attributes( def listify( - collection: types.Callable[[types.Iterable[_T]], types.Collection[_T]] = - list, + collection: types.Callable[ + [types.Iterable[_T]], types.Collection[_T] + ] = list, allow_empty: bool = True, ) -> types.Callable[ [types.Callable[..., types.Optional[types.Iterable[_T]]]], @@ -116,8 +118,9 @@ def listify( def _listify( function: types.Callable[..., types.Optional[types.Iterable[_T]]], ) -> types.Callable[..., types.Collection[_T]]: - def __listify(*args: types.Any, **kwargs: types.Any) \ - -> types.Collection[_T]: + def __listify( + *args: types.Any, **kwargs: types.Any + ) -> types.Collection[_T]: result: types.Optional[types.Iterable[_T]] = function( *args, **kwargs ) @@ -137,7 +140,9 @@ def __listify(*args: types.Any, **kwargs: types.Any) \ return _listify -def sample(sample_rate: float) -> types.Callable[ +def sample( + sample_rate: float, +) -> types.Callable[ [types.Callable[_P, _T]], types.Callable[_P, types.Optional[_T]], ]: diff --git a/python_utils/exceptions.py b/python_utils/exceptions.py index fe6750e..ee7c195 100644 --- a/python_utils/exceptions.py +++ b/python_utils/exceptions.py @@ -9,6 +9,7 @@ reraise(*args, **kwargs): Reraises the current exception. """ + from . import types diff --git a/python_utils/formatters.py b/python_utils/formatters.py index ac1d03d..4e634e4 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -13,6 +13,7 @@ default: str = 'just now') -> str: Returns string representing 'time since' e.g. 3 days ago, 5 hours ago. """ + # pyright: reportUnnecessaryIsInstance=false import datetime diff --git a/python_utils/generators.py b/python_utils/generators.py index 8ff06af..0c4a437 100644 --- a/python_utils/generators.py +++ b/python_utils/generators.py @@ -10,6 +10,7 @@ batcher(iterable, batch_size=10): Generator wrapper that returns items with a given batch size. """ + import asyncio import time diff --git a/python_utils/import_.py b/python_utils/import_.py index f1201a8..38336eb 100644 --- a/python_utils/import_.py +++ b/python_utils/import_.py @@ -11,6 +11,7 @@ Imports the requested items into the global scope, with support for relative imports and custom exception handling. """ + from . import types diff --git a/python_utils/logger.py b/python_utils/logger.py index d63d0df..bd988c5 100644 --- a/python_utils/logger.py +++ b/python_utils/logger.py @@ -24,6 +24,7 @@ >>> my_class.exception('exception') >>> my_class.log(0, 'log') """ + import abc import logging @@ -327,7 +328,7 @@ class Logged(LoggerBase): def __get_name(cls, *name_parts: str) -> str: return types.cast( str, - LoggerBase._LoggerBase__get_name(*name_parts) # type: ignore[attr-defined] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue] + LoggerBase._LoggerBase__get_name(*name_parts), # type: ignore[attr-defined] # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType, reportAttributeAccessIssue] ) def __new__(cls, *args: types.Any, **kwargs: types.Any) -> 'Logged': diff --git a/python_utils/loguru.py b/python_utils/loguru.py index 9352ffd..c1cd8ab 100644 --- a/python_utils/loguru.py +++ b/python_utils/loguru.py @@ -13,6 +13,7 @@ >>> my_class = MyClass() >>> my_class.logger.info('This is an info message') """ + from __future__ import annotations import typing diff --git a/python_utils/terminal.py b/python_utils/terminal.py index af611cd..c87a6ca 100644 --- a/python_utils/terminal.py +++ b/python_utils/terminal.py @@ -11,6 +11,7 @@ Usage example: >>> width, height = get_terminal_size() """ + from __future__ import annotations import contextlib @@ -94,7 +95,10 @@ def get_terminal_size() -> Dimensions: # pragma: no cover def _get_terminal_size_windows() -> OptionalDimensions: # pragma: no cover res = None try: - from ctypes import create_string_buffer, windll # type: ignore[attr-defined] + from ctypes import ( # type: ignore[attr-defined] + create_string_buffer, + windll, + ) # stdin handle is -10 # stdout handle is -11 @@ -153,10 +157,13 @@ def ioctl_gwinsz(fd: int) -> tuple[str, str] | None: import struct import termios - return typing.cast(_OptionalStrDimensions, struct.unpack( - 'hh', - fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'), # type: ignore[call-overload] - )) + return typing.cast( + _OptionalStrDimensions, + struct.unpack( + 'hh', + fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'), # type: ignore[call-overload] + ), + ) except Exception: return None diff --git a/setup.py b/setup.py index ba77b56..74f4a55 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ requirements and optional dependencies for different use cases such as logging, documentation, and testing. """ + import pathlib import setuptools From d64d268f41414b575f03744711af28c45c24f496 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 25 Sep 2024 03:49:53 +0200 Subject: [PATCH 120/132] protocols dont need test coverage --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 61d9675..5e83ea1 100644 --- a/.coveragerc +++ b/.coveragerc @@ -26,3 +26,4 @@ exclude_lines = @overload @types.overload @typing.overload + types.Protocol From f4629d821e537e8f9d8a4f9f7fd4e7a128c9dc66 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 25 Sep 2024 03:53:40 +0200 Subject: [PATCH 121/132] made tests slightly less flakey --- _python_utils_tests/test_time.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index a0f5c77..6f93baa 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -46,8 +46,8 @@ async def test_aio_timeout_generator( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ (0.01, 0.006, 0.5, 0.01, 'abc', 'c'), - (0.01, 0.006, 0.5, 0.01, itertools.count, 2), - (0.01, 0.006, 0.5, 0.01, itertools.count(), 2), + (0.01, 0.005, 0.5, 0.01, itertools.count, 3), + (0.01, 0.005, 0.5, 0.01, itertools.count(), 3), (0.01, 0.006, 1.0, None, 'abc', 'c'), ( timedelta(seconds=0.01), From 46c285434ad4e64c24e21efcd3eedfbb6da8d1a8 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 25 Sep 2024 04:00:23 +0200 Subject: [PATCH 122/132] more test tuning... timings are hard to test --- _python_utils_tests/test_time.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 6f93baa..b08adc8 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -46,8 +46,8 @@ async def test_aio_timeout_generator( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ (0.01, 0.006, 0.5, 0.01, 'abc', 'c'), - (0.01, 0.005, 0.5, 0.01, itertools.count, 3), - (0.01, 0.005, 0.5, 0.01, itertools.count(), 3), + (0.01, 0.007, 0.5, 0.01, itertools.count, 2), + (0.01, 0.007, 0.5, 0.01, itertools.count(), 2), (0.01, 0.006, 1.0, None, 'abc', 'c'), ( timedelta(seconds=0.01), From 8750538a016c7124d936b13b15e1fb3bb7c5635b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 25 Sep 2024 04:04:32 +0200 Subject: [PATCH 123/132] modernized github actions --- .github/workflows/main.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ea0db00..206b6b6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,9 +14,11 @@ jobs: python-version: ['3.9', '3.10', '3.11', '3.12'] # Maybe soon?, '3.13'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + with: + fetch-depth: 1 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -36,9 +38,9 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 2 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install dependencies From 108c0f5e2c7459367afa5efbdad079274fb0e34d Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Wed, 25 Sep 2024 04:10:01 +0200 Subject: [PATCH 124/132] Incrementing version to v3.9.0 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 4689e2b..8db9614 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -19,4 +19,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.8.2' +__version__ = '3.9.0' From 2e3515601c9f2c4824dcc5f8625f50c22de8e953 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Fri, 4 Oct 2024 14:42:18 +0200 Subject: [PATCH 125/132] enabled pypy3 in the tests again --- .github/workflows/main.yml | 2 +- pyproject.toml | 1 + tox.ini | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 206b6b6..5ebde9e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 4 strategy: matrix: - python-version: ['3.9', '3.10', '3.11', '3.12'] # Maybe soon?, '3.13'] + python-version: ['pypy3', '3.9', '3.10', '3.11', '3.12'] # Maybe soon?, '3.13'] steps: - uses: actions/checkout@v4 diff --git a/pyproject.toml b/pyproject.toml index f1400f8..0bf3c24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ pythonVersion = '3.9' [tool.mypy] strict = true check_untyped_defs = true +files = ['python_utils', '_python_utils_tests', 'setup.py'] [[tool.mypy.overrides]] module = '_python_utils_tests.*' diff --git a/tox.ini b/tox.ini index 1c8148e..03f2717 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = ruff, black, py39, py310, py311, py312, py313, docs, mypy, pyright +envlist = ruff, black, pypy3, py39, py310, py311, py312, py313, docs, mypy, pyright skip_missing_interpreters = True [testenv] @@ -9,6 +9,7 @@ basepython = py311: python3.11 py312: python3.12 py313: python3.13 + pypy3: pypy3 setenv = PY_IGNORE_IMPORTMISMATCH=1 deps = From 1b53d7ec9e9d8613a5dc917bc7738acf4b87f095 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 26 Oct 2024 19:17:33 +0200 Subject: [PATCH 126/132] testing reliability improvements for #42 --- _python_utils_tests/test_time.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index b08adc8..3d1f3ba 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -45,15 +45,15 @@ async def test_aio_timeout_generator( @pytest.mark.parametrize( 'timeout,interval,interval_multiplier,maximum_interval,iterable,result', [ - (0.01, 0.006, 0.5, 0.01, 'abc', 'c'), - (0.01, 0.007, 0.5, 0.01, itertools.count, 2), - (0.01, 0.007, 0.5, 0.01, itertools.count(), 2), - (0.01, 0.006, 1.0, None, 'abc', 'c'), + (0.1, 0.06, 0.5, 0.1, 'abc', 'c'), + (0.1, 0.07, 0.5, 0.1, itertools.count, 2), + (0.1, 0.07, 0.5, 0.1, itertools.count(), 2), + (0.1, 0.06, 1.0, None, 'abc', 'c'), ( - timedelta(seconds=0.01), - timedelta(seconds=0.006), + timedelta(seconds=0.1), + timedelta(seconds=0.06), 2.0, - timedelta(seconds=0.01), + timedelta(seconds=0.1), itertools.count, 2, ), From 9b971cfeba63086fd71d946dda8c48980296f91e Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Sat, 26 Oct 2024 23:22:02 +0200 Subject: [PATCH 127/132] testing reliability improvements for #42 --- _python_utils_tests/test_time.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/_python_utils_tests/test_time.py b/_python_utils_tests/test_time.py index 3d1f3ba..d4c4658 100644 --- a/_python_utils_tests/test_time.py +++ b/_python_utils_tests/test_time.py @@ -91,28 +91,28 @@ async def test_aio_generator_timeout_detector() -> None: async def generator() -> types.AsyncGenerator[int, None]: for i in range(10): - await asyncio.sleep(i / 100.0) + await asyncio.sleep(i / 20.0) yield i detector = python_utils.aio_generator_timeout_detector # Test regular timeout with reraise with pytest.raises(asyncio.TimeoutError): - async for i in detector(generator(), 0.05): + async for i in detector(generator(), 0.25): pass # Test regular timeout with clean exit - async for i in detector(generator(), 0.05, on_timeout=None): + async for i in detector(generator(), 0.25, on_timeout=None): pass assert i == 4 # Test total timeout with reraise with pytest.raises(asyncio.TimeoutError): - async for i in detector(generator(), total_timeout=0.1): + async for i in detector(generator(), total_timeout=0.5): pass # Test total timeout with clean exit - async for i in detector(generator(), total_timeout=0.1, on_timeout=None): + async for i in detector(generator(), total_timeout=0.5, on_timeout=None): pass assert i == 4 From c70383f033c1f483514c61eb603645ecb6640ca5 Mon Sep 17 00:00:00 2001 From: Zeev Rotshtein Date: Wed, 6 Nov 2024 15:28:25 +0200 Subject: [PATCH 128/132] Fix python_requires and bump version --- python_utils/__about__.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 8db9614..a96a6fa 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -19,4 +19,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.9.0' +__version__ = '3.9.1' diff --git a/setup.py b/setup.py index 74f4a55..8a66f9d 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ if __name__ == '__main__': setuptools.setup( - python_requires='>3.9.0', + python_requires='>=3.9.0', name='python-utils', version=about['__version__'], author=about['__author__'], From 05a51822ff553c920cebba15a995c718154494fd Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 26 Nov 2024 01:24:43 +0100 Subject: [PATCH 129/132] ruff fixes --- python_utils/__about__.py | 2 +- python_utils/__init__.py | 60 ++++++------- python_utils/formatters.py | 4 +- python_utils/types.py | 168 ++++++++++++++++++------------------- ruff.toml | 15 +++- 5 files changed, 128 insertions(+), 121 deletions(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index a96a6fa..8db9614 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -19,4 +19,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.9.1' +__version__ = '3.9.0' diff --git a/python_utils/__init__.py b/python_utils/__init__.py index 9a6c629..7c4242c 100644 --- a/python_utils/__init__.py +++ b/python_utils/__init__.py @@ -83,44 +83,44 @@ ) __all__ = [ + 'CastedDict', + 'LazyCastedDict', + 'Logged', + 'LoggerBase', + 'UniqueList', + 'abatcher', + 'acount', 'aio', - 'generators', + 'aio_generator_timeout_detector', + 'aio_generator_timeout_detector_decorator', + 'aio_timeout_generator', + 'batcher', + 'camel_to_underscore', 'converters', 'decorators', + 'delta_to_seconds', + 'delta_to_seconds_or_none', + 'format_time', 'formatters', + 'generators', + 'get_terminal_size', 'import_', + 'import_global', + 'listify', 'logger', - 'terminal', - 'time', - 'types', - 'to_int', - 'to_float', - 'to_unicode', - 'to_str', - 'scale_1024', + 'raise_exception', 'remap', + 'reraise', + 'scale_1024', 'set_attributes', - 'listify', - 'camel_to_underscore', - 'timesince', - 'import_global', - 'get_terminal_size', + 'terminal', + 'time', 'timedelta_to_seconds', - 'format_time', 'timeout_generator', - 'acount', - 'abatcher', - 'batcher', - 'aio_timeout_generator', - 'aio_generator_timeout_detector_decorator', - 'aio_generator_timeout_detector', - 'delta_to_seconds', - 'delta_to_seconds_or_none', - 'reraise', - 'raise_exception', - 'Logged', - 'LoggerBase', - 'CastedDict', - 'LazyCastedDict', - 'UniqueList', + 'timesince', + 'to_float', + 'to_int', + 'to_str', + 'to_unicode', + 'types', ] diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 4e634e4..0f7796b 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -163,9 +163,9 @@ def timesince( for period, singular, plural in periods: if int(period): if int(period) == 1: - output.append('%d %s' % (period, singular)) + output.append(f'{period:d} {singular}') else: - output.append('%d %s' % (period, plural)) + output.append(f'{period:d} {plural}') if output: return f'{" and ".join(output[:2])} ago' diff --git a/python_utils/types.py b/python_utils/types.py index 90bcd2a..ab89c43 100644 --- a/python_utils/types.py +++ b/python_utils/types.py @@ -54,128 +54,128 @@ ] __all__ = [ - 'OptionalScope', - 'Number', - 'DecimalNumber', - 'delta_type', - 'timestamp_type', + 'IO', + 'TYPE_CHECKING', + # ABCs (from collections.abc). + 'AbstractSet', # The types from the typing module. # Super-special typing primitives. 'Annotated', 'Any', + # One-off things. + 'AnyStr', + 'AsyncContextManager', + 'AsyncGenerator', + 'AsyncGeneratorType', + 'AsyncIterable', + 'AsyncIterator', + 'Awaitable', + # Other concrete types. + 'BinaryIO', + 'BuiltinFunctionType', + 'BuiltinMethodType', + 'ByteString', 'Callable', + # Concrete collection types. + 'ChainMap', + 'ClassMethodDescriptorType', 'ClassVar', + 'CodeType', + 'Collection', 'Concatenate', + 'Container', + 'ContextManager', + 'Coroutine', + 'CoroutineType', + 'Counter', + 'DecimalNumber', + 'DefaultDict', + 'Deque', + 'Dict', + 'DynamicClassAttribute', 'Final', 'ForwardRef', + 'FrameType', + 'FrozenSet', + # Types from the `types` module. + 'FunctionType', + 'Generator', + 'GeneratorType', 'Generic', - 'Literal', - 'SupportsIndex', - 'Optional', - 'ParamSpec', - 'ParamSpecArgs', - 'ParamSpecKwargs', - 'Protocol', - 'Tuple', - 'Type', - 'TypeVar', - 'Union', - # ABCs (from collections.abc). - 'AbstractSet', - 'ByteString', - 'Container', - 'ContextManager', + 'GetSetDescriptorType', 'Hashable', 'ItemsView', 'Iterable', 'Iterator', 'KeysView', + 'LambdaType', + 'List', + 'Literal', 'Mapping', + 'MappingProxyType', 'MappingView', + 'Match', + 'MemberDescriptorType', + 'MethodDescriptorType', + 'MethodType', + 'MethodWrapperType', + 'ModuleType', 'MutableMapping', 'MutableSequence', 'MutableSet', - 'Sequence', - 'Sized', - 'ValuesView', - 'Awaitable', - 'AsyncIterator', - 'AsyncIterable', - 'Coroutine', - 'Collection', - 'AsyncGenerator', - 'AsyncContextManager', + 'NamedTuple', # Not really a type. + 'NewType', + 'NoReturn', + 'Number', + 'Optional', + 'OptionalScope', + 'OrderedDict', + 'ParamSpec', + 'ParamSpecArgs', + 'ParamSpecKwargs', + 'Pattern', + 'Protocol', # Structural checks, a.k.a. protocols. 'Reversible', + 'Sequence', + 'Set', + 'SimpleNamespace', + 'Sized', 'SupportsAbs', 'SupportsBytes', 'SupportsComplex', 'SupportsFloat', 'SupportsIndex', + 'SupportsIndex', 'SupportsInt', 'SupportsRound', - # Concrete collection types. - 'ChainMap', - 'Counter', - 'Deque', - 'Dict', - 'DefaultDict', - 'List', - 'OrderedDict', - 'Set', - 'FrozenSet', - 'NamedTuple', # Not really a type. - 'TypedDict', # Not really a type. - 'Generator', - # Other concrete types. - 'BinaryIO', - 'IO', - 'Match', - 'Pattern', + 'Text', 'TextIO', - # One-off things. - 'AnyStr', + 'TracebackType', + 'TracebackType', + 'Tuple', + 'Type', + 'TypeAlias', + 'TypeGuard', + 'TypeVar', + 'TypedDict', # Not really a type. + 'Union', + 'ValuesView', + 'WrapperDescriptorType', 'cast', + 'coroutine', + 'delta_type', 'final', 'get_args', 'get_origin', 'get_type_hints', 'is_typeddict', - 'NewType', + 'new_class', 'no_type_check', 'no_type_check_decorator', - 'NoReturn', 'overload', - 'runtime_checkable', - 'Text', - 'TYPE_CHECKING', - 'TypeAlias', - 'TypeGuard', - 'TracebackType', - # Types from the `types` module. - 'FunctionType', - 'LambdaType', - 'CodeType', - 'MappingProxyType', - 'SimpleNamespace', - 'GeneratorType', - 'CoroutineType', - 'AsyncGeneratorType', - 'MethodType', - 'BuiltinFunctionType', - 'BuiltinMethodType', - 'WrapperDescriptorType', - 'MethodWrapperType', - 'MethodDescriptorType', - 'ClassMethodDescriptorType', - 'ModuleType', - 'TracebackType', - 'FrameType', - 'GetSetDescriptorType', - 'MemberDescriptorType', - 'new_class', - 'resolve_bases', 'prepare_class', - 'DynamicClassAttribute', - 'coroutine', + 'resolve_bases', + 'runtime_checkable', + 'timestamp_type', ] diff --git a/ruff.toml b/ruff.toml index 1ce08a4..294cb3c 100644 --- a/ruff.toml +++ b/ruff.toml @@ -6,10 +6,15 @@ target-version = 'py39' exclude = [ '.venv', '.tox', + # Ignore local test files/directories/old-stuff 'test.py', + '*_old.py', ] -lint.ignore = [ +line-length = 79 + +[lint] +ignore = [ 'A001', # Variable {name} is shadowing a Python builtin 'A002', # Argument {name} is shadowing a Python builtin 'A003', # Class attribute {name} is shadowing a Python builtin @@ -27,13 +32,14 @@ lint.ignore = [ 'RET506', # Unnecessary `else` after `raise` statement 'Q001', # Remove bad quotes 'Q002', # Remove bad quotes + 'FA100', # Missing `from __future__ import annotations`, but uses `typing.Optional` 'COM812', # Missing trailing comma in a list 'ISC001', # String concatenation with implicit str conversion 'SIM108', # Ternary operators are not always more readable - 'RUF100', # Unused `noqa` directive. These vary per Python version so this warning is often incorrect. + 'RUF100', # Unused noqa directives. Due to multiple Python versions, we need to keep them ] -line-length = 79 -lint.select = [ + +select = [ 'A', # flake8-builtins 'ASYNC', # flake8 async checker 'B', # flake8-bugbear @@ -105,3 +111,4 @@ max-line-length = 79 [lint.flake8-pytest-style] mark-parentheses = true + From 485059617de23a7aeb31f67e81493346f9bd1e41 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 26 Nov 2024 01:28:21 +0100 Subject: [PATCH 130/132] fixing github actions for pypy --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5ebde9e..0e1f36c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ jobs: timeout-minutes: 4 strategy: matrix: - python-version: ['pypy3', '3.9', '3.10', '3.11', '3.12'] # Maybe soon?, '3.13'] + python-version: ['pypy3.9', 'pypy3.10', '3.9', '3.10', '3.11', '3.12'] # Maybe soon?, '3.13'] steps: - uses: actions/checkout@v4 From 723efcae9fc8be8e8c49959f155f821f67148c99 Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 26 Nov 2024 01:36:38 +0100 Subject: [PATCH 131/132] made ruff happy without breaking the code --- python_utils/formatters.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python_utils/formatters.py b/python_utils/formatters.py index 0f7796b..44ec873 100644 --- a/python_utils/formatters.py +++ b/python_utils/formatters.py @@ -161,11 +161,11 @@ def timesince( output: types.List[str] = [] for period, singular, plural in periods: - if int(period): - if int(period) == 1: - output.append(f'{period:d} {singular}') - else: - output.append(f'{period:d} {plural}') + int_period = int(period) + if int_period == 1: + output.append(f'{int_period} {singular}') + elif int_period: + output.append(f'{int_period} {plural}') if output: return f'{" and ".join(output[:2])} ago' From 0a76c1c1ddea61d98ae49c1ada7b6dac206e3e6b Mon Sep 17 00:00:00 2001 From: Rick van Hattem Date: Tue, 26 Nov 2024 01:38:38 +0100 Subject: [PATCH 132/132] Incrementing version to v3.9.1 --- python_utils/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_utils/__about__.py b/python_utils/__about__.py index 8db9614..a96a6fa 100644 --- a/python_utils/__about__.py +++ b/python_utils/__about__.py @@ -19,4 +19,4 @@ ) __url__: str = 'https://github.com/WoLpH/python-utils' # Omit type info due to automatic versioning script -__version__ = '3.9.0' +__version__ = '3.9.1'