diff --git a/Lib/configparser.py b/Lib/configparser.py index af5aca1fea..df2d7e335d 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -19,36 +19,37 @@ inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section='DEFAULT', interpolation=, converters=): - Create the parser. When `defaults' is given, it is initialized into the + + Create the parser. When `defaults` is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. - When `dict_type' is given, it will be used to create the dictionary + When `dict_type` is given, it will be used to create the dictionary objects for the list of sections, for the options within a section, and for the default values. - When `delimiters' is given, it will be used as the set of substrings + When `delimiters` is given, it will be used as the set of substrings that divide keys from values. - When `comment_prefixes' is given, it will be used as the set of + When `comment_prefixes` is given, it will be used as the set of substrings that prefix comments in empty lines. Comments can be indented. - When `inline_comment_prefixes' is given, it will be used as the set of + When `inline_comment_prefixes` is given, it will be used as the set of substrings that prefix comments in non-empty lines. When `strict` is True, the parser won't allow for any section or option duplicates while reading from a single source (file, string or dictionary). Default is True. - When `empty_lines_in_values' is False (default: True), each empty line + When `empty_lines_in_values` is False (default: True), each empty line marks the end of an option. Otherwise, internal empty lines of a multiline option are kept as part of the value. - When `allow_no_value' is True (default: False), options without + When `allow_no_value` is True (default: False), options without values are accepted; the value presented for these is None. - When `default_section' is given, the name of the special section is + When `default_section` is given, the name of the special section is named accordingly. By default it is called ``"DEFAULT"`` but this can be customized to point to any other valid section name. Its current value can be retrieved using the ``parser_instance.default_section`` @@ -56,7 +57,7 @@ When `interpolation` is given, it should be an Interpolation subclass instance. It will be used as the handler for option value - pre-processing when using getters. RawConfigParser object s don't do + pre-processing when using getters. RawConfigParser objects don't do any sort of interpolation, whereas ConfigParser uses an instance of BasicInterpolation. The library also provides a ``zc.buildbot`` inspired ExtendedInterpolation implementation. @@ -80,14 +81,14 @@ Return list of configuration options for the named section. read(filenames, encoding=None) - Read and parse the list of named configuration files, given by + Read and parse the iterable of named configuration files, given by name. A single filename is also allowed. Non-existing files are ignored. Return list of successfully read files. read_file(f, filename=None) Read and parse one configuration file, given as a file object. The filename defaults to f.name; it is only used in error - messages (if f has no `name' attribute, the string `' is used). + messages (if f has no `name` attribute, the string `` is used). read_string(string) Read configuration from a given string. @@ -103,9 +104,9 @@ Return a string value for the named option. All % interpolations are expanded in the return values, based on the defaults passed into the constructor and the DEFAULT section. Additional substitutions may be - provided using the `vars' argument, which must be a dictionary whose - contents override any pre-existing defaults. If `option' is a key in - `vars', the value from `vars' is used. + provided using the `vars` argument, which must be a dictionary whose + contents override any pre-existing defaults. If `option` is a key in + `vars`, the value from `vars` is used. getint(section, options, raw=False, vars=None, fallback=_UNSET) Like get(), but convert value to an integer. @@ -134,15 +135,16 @@ write(fp, space_around_delimiters=True) Write the configuration state in .ini format. If - `space_around_delimiters' is True (the default), delimiters + `space_around_delimiters` is True (the default), delimiters between keys and values are surrounded by spaces. """ from collections.abc import MutableMapping -from collections import OrderedDict as _default_dict, ChainMap as _ChainMap +from collections import ChainMap as _ChainMap import functools import io import itertools +import os import re import sys import warnings @@ -156,6 +158,7 @@ "LegacyInterpolation", "SectionProxy", "ConverterMapping", "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH"] +_default_dict = dict DEFAULTSECT = "DEFAULT" MAX_INTERPOLATION_DEPTH = 10 @@ -314,7 +317,7 @@ def __init__(self, source=None, filename=None): def filename(self): """Deprecated, use `source'.""" warnings.warn( - "The 'filename' attribute will be removed in future versions. " + "The 'filename' attribute will be removed in Python 3.12. " "Use 'source' instead.", DeprecationWarning, stacklevel=2 ) @@ -324,7 +327,7 @@ def filename(self): def filename(self, value): """Deprecated, user `source'.""" warnings.warn( - "The 'filename' attribute will be removed in future versions. " + "The 'filename' attribute will be removed in Python 3.12. " "Use 'source' instead.", DeprecationWarning, stacklevel=2 ) @@ -350,7 +353,7 @@ def __init__(self, filename, lineno, line): # Used in parser getters to indicate the default behaviour when a specific -# option is not found it to raise an exception. Created to enable `None' as +# option is not found it to raise an exception. Created to enable `None` as # a valid fallback value. _UNSET = object() @@ -384,7 +387,7 @@ class BasicInterpolation(Interpolation): would resolve the "%(dir)s" to the value of dir. All reference expansions are done late, on demand. If a user needs to use a bare % in a configuration file, she can escape it by writing %%. Other % usage - is considered a user error and raises `InterpolationSyntaxError'.""" + is considered a user error and raises `InterpolationSyntaxError`.""" _KEYCRE = re.compile(r"%\(([^)]+)\)s") @@ -445,7 +448,7 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, class ExtendedInterpolation(Interpolation): """Advanced variant of interpolation, supports the syntax used by - `zc.buildout'. Enables interpolation between sections.""" + `zc.buildout`. Enables interpolation between sections.""" _KEYCRE = re.compile(r"\$\{([^}]+)\}") @@ -523,6 +526,15 @@ class LegacyInterpolation(Interpolation): _KEYCRE = re.compile(r"%\(([^)]*)\)s|.") + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + warnings.warn( + "LegacyInterpolation has been deprecated since Python 3.2 " + "and will be removed from the configparser module in Python 3.13. " + "Use BasicInterpolation or ExtendedInterpolation instead.", + DeprecationWarning, stacklevel=2 + ) + def before_get(self, parser, section, option, value, vars): rawval = value depth = MAX_INTERPOLATION_DEPTH @@ -561,7 +573,7 @@ class RawConfigParser(MutableMapping): # Regular expressions for parsing section headers and options _SECT_TMPL = r""" \[ # [ - (?P
[^]]+) # very permissive! + (?P
.+) # very permissive! \] # ] """ _OPT_TMPL = r""" @@ -609,9 +621,6 @@ def __init__(self, defaults=None, dict_type=_default_dict, self._converters = ConverterMapping(self) self._proxies = self._dict() self._proxies[default_section] = SectionProxy(self, default_section) - if defaults: - for key, value in defaults.items(): - self._defaults[self.optionxform(key)] = value self._delimiters = tuple(delimiters) if delimiters == ('=', ':'): self._optcre = self.OPTCRE_NV if allow_no_value else self.OPTCRE @@ -634,8 +643,15 @@ def __init__(self, defaults=None, dict_type=_default_dict, self._interpolation = self._DEFAULT_INTERPOLATION if self._interpolation is None: self._interpolation = Interpolation() + if not isinstance(self._interpolation, Interpolation): + raise TypeError( + f"interpolation= must be None or an instance of Interpolation;" + f" got an object of type {type(self._interpolation)}" + ) if converters is not _UNSET: self._converters.update(converters) + if defaults: + self._read_defaults(defaults) def defaults(self): return self._defaults @@ -676,19 +692,20 @@ def options(self, section): return list(opts.keys()) def read(self, filenames, encoding=None): - """Read and parse a filename or a list of filenames. + """Read and parse a filename or an iterable of filenames. Files that cannot be opened are silently ignored; this is - designed so that you can specify a list of potential + designed so that you can specify an iterable of potential configuration file locations (e.g. current directory, user's home directory, systemwide directory), and all existing - configuration files in the list will be read. A single + configuration files in the iterable will be read. A single filename may also be given. Return list of successfully read files. """ - if isinstance(filenames, str): + if isinstance(filenames, (str, bytes, os.PathLike)): filenames = [filenames] + encoding = io.text_encoding(encoding) read_ok = [] for filename in filenames: try: @@ -696,16 +713,18 @@ def read(self, filenames, encoding=None): self._read(fp, filename) except OSError: continue + if isinstance(filename, os.PathLike): + filename = os.fspath(filename) read_ok.append(filename) return read_ok def read_file(self, f, source=None): """Like read() but the argument must be a file-like object. - The `f' argument must be iterable, returning one line at a time. - Optional second argument is the `source' specifying the name of the - file being read. If not given, it is taken from f.name. If `f' has no - `name' attribute, `' is used. + The `f` argument must be iterable, returning one line at a time. + Optional second argument is the `source` specifying the name of the + file being read. If not given, it is taken from f.name. If `f` has no + `name` attribute, `` is used. """ if source is None: try: @@ -729,7 +748,7 @@ def read_dict(self, dictionary, source=''): All types held in the dictionary are converted to strings during reading, including section names, option names and keys. - Optional second argument is the `source' specifying the name of the + Optional second argument is the `source` specifying the name of the dictionary being read. """ elements_added = set() @@ -753,7 +772,7 @@ def read_dict(self, dictionary, source=''): def readfp(self, fp, filename=None): """Deprecated, use read_file instead.""" warnings.warn( - "This method will be removed in future versions. " + "This method will be removed in Python 3.12. " "Use 'parser.read_file()' instead.", DeprecationWarning, stacklevel=2 ) @@ -762,15 +781,15 @@ def readfp(self, fp, filename=None): def get(self, section, option, *, raw=False, vars=None, fallback=_UNSET): """Get an option value for a given section. - If `vars' is provided, it must be a dictionary. The option is looked up - in `vars' (if provided), `section', and in `DEFAULTSECT' in that order. - If the key is not found and `fallback' is provided, it is used as - a fallback value. `None' can be provided as a `fallback' value. + If `vars` is provided, it must be a dictionary. The option is looked up + in `vars` (if provided), `section`, and in `DEFAULTSECT` in that order. + If the key is not found and `fallback` is provided, it is used as + a fallback value. `None` can be provided as a `fallback` value. - If interpolation is enabled and the optional argument `raw' is False, + If interpolation is enabled and the optional argument `raw` is False, all interpolations are expanded in the return values. - Arguments `raw', `vars', and `fallback' are keyword only. + Arguments `raw`, `vars`, and `fallback` are keyword only. The section DEFAULT is special. """ @@ -830,8 +849,8 @@ def items(self, section=_UNSET, raw=False, vars=None): All % interpolations are expanded in the return values, based on the defaults passed into the constructor, unless the optional argument - `raw' is true. Additional substitutions may be provided using the - `vars' argument, which must be a dictionary whose contents overrides + `raw` is true. Additional substitutions may be provided using the + `vars` argument, which must be a dictionary whose contents overrides any pre-existing defaults. The section DEFAULT is special. @@ -844,6 +863,7 @@ def items(self, section=_UNSET, raw=False, vars=None): except KeyError: if section != self.default_section: raise NoSectionError(section) + orig_keys = list(d.keys()) # Update with the entry specific variables if vars: for key, value in vars.items(): @@ -852,7 +872,7 @@ def items(self, section=_UNSET, raw=False, vars=None): section, option, d[option], d) if raw: value_getter = lambda option: d[option] - return [(option, value_getter(option)) for option in d.keys()] + return [(option, value_getter(option)) for option in orig_keys] def popitem(self): """Remove a section from the parser and return it as @@ -872,8 +892,8 @@ def optionxform(self, optionstr): def has_option(self, section, option): """Check for the existence of a given option in a given section. - If the specified `section' is None or an empty string, DEFAULT is - assumed. If the specified `section' does not exist, returns False.""" + If the specified `section` is None or an empty string, DEFAULT is + assumed. If the specified `section` does not exist, returns False.""" if not section or section == self.default_section: option = self.optionxform(option) return option in self._defaults @@ -901,8 +921,11 @@ def set(self, section, option, value=None): def write(self, fp, space_around_delimiters=True): """Write an .ini-format representation of the configuration state. - If `space_around_delimiters' is True (the default), delimiters + If `space_around_delimiters` is True (the default), delimiters between keys and values are surrounded by spaces. + + Please note that comments in the original configuration file are not + preserved when writing the configuration back. """ if space_around_delimiters: d = " {} ".format(self._delimiters[0]) @@ -916,7 +939,7 @@ def write(self, fp, space_around_delimiters=True): self._sections[section].items(), d) def _write_section(self, fp, section_name, section_items, delimiter): - """Write a single section to the specified `fp'.""" + """Write a single section to the specified `fp`.""" fp.write("[{}]\n".format(section_name)) for key, value in section_items: value = self._interpolation.before_write(self, section_name, key, @@ -959,7 +982,8 @@ def __getitem__(self, key): def __setitem__(self, key, value): # To conform with the mapping protocol, overwrites existing values in # the section. - + if key in self and self[key] is value: + return # XXX this is not atomic if read_dict fails at any point. Then again, # no update method in configparser is atomic in this implementation. if key == self.default_section: @@ -989,8 +1013,8 @@ def _read(self, fp, fpname): """Parse a sectioned configuration file. Each section in a configuration file contains a header, indicated by - a name in square brackets (`[]'), plus key/value options, indicated by - `name' and `value' delimited with a specific substring (`=' or `:' by + a name in square brackets (`[]`), plus key/value options, indicated by + `name` and `value` delimited with a specific substring (`=` or `:` by default). Values can span multiple lines, as long as they are indented deeper @@ -998,9 +1022,9 @@ def _read(self, fp, fpname): lines may be treated as parts of multiline values or ignored. Configuration files may include comments, prefixed by specific - characters (`#' and `;' by default). Comments may appear on their own + characters (`#` and `;` by default). Comments may appear on their own in an otherwise empty line or may be entered in lines holding values or - section names. + section names. Please note that comments get stripped off when reading configuration files. """ elements_added = set() cursect = None # None, or a dictionary @@ -1119,6 +1143,12 @@ def _join_multiline_values(self): section, name, val) + def _read_defaults(self, defaults): + """Read the defaults passed in the initializer. + Note: values can be non-string.""" + for key, value in defaults.items(): + self._defaults[self.optionxform(key)] = value + def _handle_error(self, exc, fpname, lineno, line): if not exc: exc = ParsingError(fpname) @@ -1135,7 +1165,7 @@ def _unify_values(self, section, vars): sectiondict = self._sections[section] except KeyError: if section != self.default_section: - raise NoSectionError(section) + raise NoSectionError(section) from None # Update with the entry specific variables vardict = {} if vars: @@ -1196,6 +1226,19 @@ def add_section(self, section): self._validate_value_types(section=section) super().add_section(section) + def _read_defaults(self, defaults): + """Reads the defaults passed in the initializer, implicitly converting + values to strings like the rest of the API. + + Does not perform interpolation for backwards compatibility. + """ + try: + hold_interpolation = self._interpolation + self._interpolation = Interpolation() + self.read_dict({self.default_section: defaults}) + finally: + self._interpolation = hold_interpolation + class SafeConfigParser(ConfigParser): """ConfigParser alias for backwards compatibility purposes.""" @@ -1204,7 +1247,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) warnings.warn( "The SafeConfigParser class has been renamed to ConfigParser " - "in Python 3.2. This alias will be removed in future versions." + "in Python 3.2. This alias will be removed in Python 3.12." " Use ConfigParser directly instead.", DeprecationWarning, stacklevel=2 ) diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 84499689e1..6d5c74ff48 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -79,6 +79,7 @@ def basic_test(self, cf): 'Spacey Bar', 'Spacey Bar From The Beginning', 'Types', + 'This One Has A ] In It', ] if self.allow_no_value: @@ -130,6 +131,7 @@ def basic_test(self, cf): eq(cf.get('Types', 'float'), "0.44") eq(cf.getboolean('Types', 'boolean'), False) eq(cf.get('Types', '123'), 'strange but acceptable') + eq(cf.get('This One Has A ] In It', 'forks'), 'spoons') if self.allow_no_value: eq(cf.get('NoValue', 'option-without-value'), None) @@ -320,6 +322,8 @@ def test_basic(self): float {0[0]} 0.44 boolean {0[0]} NO 123 {0[1]} strange but acceptable +[This One Has A ] In It] + forks {0[0]} spoons """.format(self.delimiters, self.comment_prefixes) if self.allow_no_value: config_string += ( @@ -394,6 +398,9 @@ def test_basic_from_dict(self): "boolean": False, 123: "strange but acceptable", }, + "This One Has A ] In It": { + "forks": "spoons" + }, } if self.allow_no_value: config.update({ @@ -709,39 +716,37 @@ class mystr(str): cf.set("sect", "option1", "splat") cf.set("sect", "option2", "splat") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_read_returns_file_list(self): if self.delimiters[0] != '=': self.skipTest('incompatible format') file1 = support.findfile("cfgparser.1") # check when we pass a mix of readable and non-readable files: cf = self.newconfig() - parsed_files = cf.read([file1, "nonexistent-file"]) + parsed_files = cf.read([file1, "nonexistent-file"], encoding="utf-8") self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only a filename: cf = self.newconfig() - parsed_files = cf.read(file1) + parsed_files = cf.read(file1, encoding="utf-8") self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only a Path object: cf = self.newconfig() - parsed_files = cf.read(pathlib.Path(file1)) + parsed_files = cf.read(pathlib.Path(file1), encoding="utf-8") self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we passed both a filename and a Path object: cf = self.newconfig() - parsed_files = cf.read([pathlib.Path(file1), file1]) + parsed_files = cf.read([pathlib.Path(file1), file1], encoding="utf-8") self.assertEqual(parsed_files, [file1, file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only missing files: cf = self.newconfig() - parsed_files = cf.read(["nonexistent-file"]) + parsed_files = cf.read(["nonexistent-file"], encoding="utf-8") self.assertEqual(parsed_files, []) # check when we pass no files: cf = self.newconfig() - parsed_files = cf.read([]) + parsed_files = cf.read([], encoding="utf-8") self.assertEqual(parsed_files, []) @unittest.skip("TODO: RUSTPYTHON, suspected to make CI hang") @@ -751,15 +756,15 @@ def test_read_returns_file_list_with_bytestring_path(self): file1_bytestring = support.findfile("cfgparser.1").encode() # check when passing an existing bytestring path cf = self.newconfig() - parsed_files = cf.read(file1_bytestring) + parsed_files = cf.read(file1_bytestring, encoding="utf-8") self.assertEqual(parsed_files, [file1_bytestring]) # check when passing an non-existing bytestring path cf = self.newconfig() - parsed_files = cf.read(b'nonexistent-file') + parsed_files = cf.read(b'nonexistent-file', encoding="utf-8") self.assertEqual(parsed_files, []) # check when passing both an existing and non-existing bytestring path cf = self.newconfig() - parsed_files = cf.read([file1_bytestring, b'nonexistent-file']) + parsed_files = cf.read([file1_bytestring, b'nonexistent-file'], encoding="utf-8") self.assertEqual(parsed_files, [file1_bytestring]) # shared by subclasses @@ -830,8 +835,6 @@ def test_clear(self): self.assertEqual(set(cf.sections()), set()) self.assertEqual(set(cf[self.default_section].keys()), {'foo'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_setitem(self): cf = self.fromstring(""" [section1] @@ -924,8 +927,6 @@ def test_interpolation_missing_value(self): self.assertEqual(e.args, ('name', 'Interpolation Error', '%(reference)s', 'reference')) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_items(self): self.check_items_config([('default', ''), ('getdefault', '||'), @@ -981,8 +982,6 @@ def test_add_section_default(self): cf = self.newconfig() self.assertRaises(ValueError, cf.add_section, self.default_section) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_defaults_keyword(self): """bpo-23835 fix for ConfigParser""" cf = self.newconfig(defaults={1: 2.4}) @@ -1031,7 +1030,9 @@ class CustomConfigParser(configparser.ConfigParser): class ConfigParserTestCaseLegacyInterpolation(ConfigParserTestCase): config_class = configparser.ConfigParser - interpolation = configparser.LegacyInterpolation() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + interpolation = configparser.LegacyInterpolation() def test_set_malformatted_interpolation(self): cf = self.fromstring("[sect]\n" @@ -1051,6 +1052,14 @@ def test_set_malformatted_interpolation(self): self.assertEqual(cf.get("sect", "option2"), "foo%%bar") +class ConfigParserTestCaseInvalidInterpolationType(unittest.TestCase): + def test_error_on_wrong_type_for_interpolation(self): + for value in [configparser.ExtendedInterpolation, 42, "a string"]: + with self.subTest(value=value): + with self.assertRaises(TypeError): + configparser.ConfigParser(interpolation=value) + + class ConfigParserTestCaseNonStandardDelimiters(ConfigParserTestCase): delimiters = (':=', '$') comment_prefixes = ('//', '"') @@ -1074,7 +1083,7 @@ def setUp(self): cf.add_section(s) for j in range(10): cf.set(s, 'lovely_spam{}'.format(j), self.wonderful_spam) - with open(os_helper.TESTFN, 'w') as f: + with open(os_helper.TESTFN, 'w', encoding="utf-8") as f: cf.write(f) def tearDown(self): @@ -1084,7 +1093,7 @@ def test_dominating_multiline_values(self): # We're reading from file because this is where the code changed # during performance updates in Python 3.2 cf_from_file = self.newconfig() - with open(os_helper.TESTFN) as f: + with open(os_helper.TESTFN, encoding="utf-8") as f: cf_from_file.read_file(f) self.assertEqual(cf_from_file.get('section8', 'lovely_spam4'), self.wonderful_spam.replace('\t\n', '\n')) @@ -1105,8 +1114,6 @@ def test_interpolation(self): eq(cf.get("Foo", "bar11"), "something %(with11)s lots of interpolation (11 steps)") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_items(self): self.check_items_config([('default', ''), ('getdefault', '|%(default)s|'), @@ -1485,7 +1492,7 @@ def fromstring(self, string, defaults=None): class FakeFile: def __init__(self): file_path = support.findfile("cfgparser.1") - with open(file_path) as f: + with open(file_path, encoding="utf-8") as f: self.lines = f.readlines() self.lines.reverse() @@ -1512,7 +1519,7 @@ def test_file(self): pass # unfortunately we can't test bytes on this path for file_path in file_paths: parser = configparser.ConfigParser() - with open(file_path) as f: + with open(file_path, encoding="utf-8") as f: parser.read_file(f) self.assertIn("Foo Bar", parser) self.assertIn("foo", parser["Foo Bar"]) @@ -1663,6 +1670,14 @@ def test_safeconfigparser_deprecation(self): for warning in w: self.assertTrue(warning.category is DeprecationWarning) + def test_legacyinterpolation_deprecation(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always", DeprecationWarning) + configparser.LegacyInterpolation() + self.assertGreaterEqual(len(w), 1) + for warning in w: + self.assertIs(warning.category, DeprecationWarning) + def test_sectionproxy_repr(self): parser = configparser.ConfigParser() parser.read_string(""" @@ -2140,8 +2155,7 @@ def test_instance_assignment(self): class MiscTestCase(unittest.TestCase): def test__all__(self): - not_exported = {"Error"} - support.check__all__(self, configparser, not_exported=not_exported) + support.check__all__(self, configparser, not_exported={"Error"}) if __name__ == '__main__':