diff --git a/.bzrignore b/.bzrignore index 2527052a1ba..abca6b1c5d6 100644 --- a/.bzrignore +++ b/.bzrignore @@ -11,6 +11,7 @@ python build Makefile.pre platform +pybuilddir.txt pyconfig.h libpython*.a libpython*.so* diff --git a/.hgignore b/.hgignore index de9bd5dbe29..c51c53cfbd7 100644 --- a/.hgignore +++ b/.hgignore @@ -35,6 +35,7 @@ Parser/pgen$ Parser/pgen.stamp$ ^core ^python-gdb.py +^pybuilddir.txt syntax: glob python.exe-gdb.py diff --git a/Demo/newmetaclasses/Eiffel.py b/Demo/newmetaclasses/Eiffel.py index 730a85da5f7..f3f116fe155 100644 --- a/Demo/newmetaclasses/Eiffel.py +++ b/Demo/newmetaclasses/Eiffel.py @@ -29,7 +29,7 @@ def convert_methods(cls, dict): pre = dict.get("%s_pre" % m) post = dict.get("%s_post" % m) if pre or post: - dict[k] = cls.make_eiffel_method(dict[m], pre, post) + dict[m] = cls.make_eiffel_method(dict[m], pre, post) class EiffelMetaClass1(EiffelBaseMetaClass): # an implementation of the "eiffel" meta class that uses nested functions diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index cb43cbfc450..32a414b1aca 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -43,7 +43,7 @@ Allocating Objects on the Heap Allocate a new Python object using the C structure type *TYPE* and the Python type object *type*. Fields not defined by the Python object header are not initialized; the object's reference count will be one. The size of - the memory allocation is determined from the :attr:`tp_basicsize` field of + the memory allocation is determined from the :c:member:`~PyTypeObject.tp_basicsize` field of the type object. @@ -52,7 +52,7 @@ Allocating Objects on the Heap Allocate a new Python object using the C structure type *TYPE* and the Python type object *type*. Fields not defined by the Python object header are not initialized. The allocated memory allows for the *TYPE* structure - plus *size* fields of the size given by the :attr:`tp_itemsize` field of + plus *size* fields of the size given by the :c:member:`~PyTypeObject.tp_itemsize` field of *type*. This is useful for implementing objects like tuples, which are able to determine their size at construction time. Embedding the array of fields into the same allocation decreases the number of allocations, @@ -67,7 +67,7 @@ Allocating Objects on the Heap Releases memory allocated to an object using :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`. This is normally called from the - :attr:`tp_dealloc` handler specified in the object's type. The fields of + :c:member:`~PyTypeObject.tp_dealloc` handler specified in the object's type. The fields of the object should not be accessed after this call as the memory is no longer a valid Python object. diff --git a/Doc/c-api/codec.rst b/Doc/c-api/codec.rst index 8207ae044d7..83252afbb7f 100644 --- a/Doc/c-api/codec.rst +++ b/Doc/c-api/codec.rst @@ -52,19 +52,19 @@ and *NULL* returned. .. c:function:: PyObject* PyCodec_IncrementalEncoder(const char *encoding, const char *errors) - Get an :class:`IncrementalEncoder` object for the given *encoding*. + Get an :class:`~codecs.IncrementalEncoder` object for the given *encoding*. .. c:function:: PyObject* PyCodec_IncrementalDecoder(const char *encoding, const char *errors) - Get an :class:`IncrementalDecoder` object for the given *encoding*. + Get an :class:`~codecs.IncrementalDecoder` object for the given *encoding*. .. c:function:: PyObject* PyCodec_StreamReader(const char *encoding, PyObject *stream, const char *errors) - Get a :class:`StreamReader` factory function for the given *encoding*. + Get a :class:`~codecs.StreamReader` factory function for the given *encoding*. .. c:function:: PyObject* PyCodec_StreamWriter(const char *encoding, PyObject *stream, const char *errors) - Get a :class:`StreamWriter` factory function for the given *encoding*. + Get a :class:`~codecs.StreamWriter` factory function for the given *encoding*. Registry API for Unicode encoding error handlers diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 025b75a70b2..91964d05737 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -192,12 +192,19 @@ is a separate error indicator for each thread. when the system call returns an error. -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) +.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObject(PyObject *type, PyObject *filenameObject) Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if - *filename* is not *NULL*, it is passed to the constructor of *type* as a third - parameter. In the case of exceptions such as :exc:`IOError` and :exc:`OSError`, - this is used to define the :attr:`filename` attribute of the exception instance. + *filenameObject* is not *NULL*, it is passed to the constructor of *type* as + a third parameter. In the case of exceptions such as :exc:`IOError` and + :exc:`OSError`, this is used to define the :attr:`filename` attribute of the + exception instance. + + +.. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) + + Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename + is given as a C string. .. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) @@ -220,14 +227,29 @@ is a separate error indicator for each thread. .. versionadded:: 2.3 -.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) +.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilenameObject(int ierr, PyObject *filenameObject) Similar to :c:func:`PyErr_SetFromWindowsErr`, with the additional behavior that - if *filename* is not *NULL*, it is passed to the constructor of + if *filenameObject* is not *NULL*, it is passed to the constructor of :exc:`WindowsError` as a third parameter. Availability: Windows. -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, char *filename) +.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) + + Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the + filename is given as a C string. Availability: Windows. + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) + + Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an + additional parameter specifying the exception type to be raised. + Availability: Windows. + + .. versionadded:: 2.3 + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, const char *filename) Similar to :c:func:`PyErr_SetFromWindowsErrWithFilename`, with an additional parameter specifying the exception type to be raised. Availability: Windows. diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index 20a7a60cddc..bbd422368f9 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -111,7 +111,8 @@ change in future releases of Python. .. index:: single: EOFError (built-in exception) Equivalent to ``p.readline([n])``, this function reads one line from the - object *p*. *p* may be a file object or any object with a :meth:`readline` + object *p*. *p* may be a file object or any object with a + :meth:`~io.IOBase.readline` method. If *n* is ``0``, exactly one line is read, regardless of the length of the line. If *n* is greater than ``0``, no more than *n* bytes will be read from the file; a partial line can be returned. In both cases, an empty string diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index 2a4fda4f3c2..b0a2d5c08dd 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -15,10 +15,10 @@ collection. .. An example showing the use of these interfaces can be found in "Supporting the .. Cycle Collector (XXX not found: ../ext/example-cycle-support.html)". -To create a container type, the :attr:`tp_flags` field of the type object must +To create a container type, the :c:member:`~PyTypeObject.tp_flags` field of the type object must include the :const:`Py_TPFLAGS_HAVE_GC` and provide an implementation of the -:attr:`tp_traverse` handler. If instances of the type are mutable, a -:attr:`tp_clear` implementation must also be provided. +:c:member:`~PyTypeObject.tp_traverse` handler. If instances of the type are mutable, a +:c:member:`~PyTypeObject.tp_clear` implementation must also be provided. .. data:: Py_TPFLAGS_HAVE_GC @@ -68,7 +68,7 @@ Constructors for container types must conform to two rules: Adds the object *op* to the set of container objects tracked by the collector. The collector can run at unexpected times so objects must be valid while being tracked. This should be called once all the fields - followed by the :attr:`tp_traverse` handler become valid, usually near the + followed by the :c:member:`~PyTypeObject.tp_traverse` handler become valid, usually near the end of the constructor. @@ -97,8 +97,8 @@ rules: Remove the object *op* from the set of container objects tracked by the collector. Note that :c:func:`PyObject_GC_Track` can be called again on this object to add it back to the set of tracked objects. The deallocator - (:attr:`tp_dealloc` handler) should call this for the object before any of - the fields used by the :attr:`tp_traverse` handler become invalid. + (:c:member:`~PyTypeObject.tp_dealloc` handler) should call this for the object before any of + the fields used by the :c:member:`~PyTypeObject.tp_traverse` handler become invalid. .. c:function:: void _PyObject_GC_UNTRACK(PyObject *op) @@ -106,19 +106,19 @@ rules: A macro version of :c:func:`PyObject_GC_UnTrack`. It should not be used for extension modules. -The :attr:`tp_traverse` handler accepts a function parameter of this type: +The :c:member:`~PyTypeObject.tp_traverse` handler accepts a function parameter of this type: .. c:type:: int (*visitproc)(PyObject *object, void *arg) - Type of the visitor function passed to the :attr:`tp_traverse` handler. + Type of the visitor function passed to the :c:member:`~PyTypeObject.tp_traverse` handler. The function should be called with an object to traverse as *object* and - the third parameter to the :attr:`tp_traverse` handler as *arg*. The + the third parameter to the :c:member:`~PyTypeObject.tp_traverse` handler as *arg*. The Python core uses several visitor functions to implement cyclic garbage detection; it's not expected that users will need to write their own visitor functions. -The :attr:`tp_traverse` handler must have the following type: +The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type: .. c:type:: int (*traverseproc)(PyObject *self, visitproc visit, void *arg) @@ -130,15 +130,15 @@ The :attr:`tp_traverse` handler must have the following type: object argument. If *visit* returns a non-zero value that value should be returned immediately. -To simplify writing :attr:`tp_traverse` handlers, a :c:func:`Py_VISIT` macro is -provided. In order to use this macro, the :attr:`tp_traverse` implementation +To simplify writing :c:member:`~PyTypeObject.tp_traverse` handlers, a :c:func:`Py_VISIT` macro is +provided. In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` implementation must name its arguments exactly *visit* and *arg*: .. c:function:: void Py_VISIT(PyObject *o) Call the *visit* callback, with arguments *o* and *arg*. If *visit* returns - a non-zero value, then return it. Using this macro, :attr:`tp_traverse` + a non-zero value, then return it. Using this macro, :c:member:`~PyTypeObject.tp_traverse` handlers look like:: static int @@ -151,7 +151,7 @@ must name its arguments exactly *visit* and *arg*: .. versionadded:: 2.4 -The :attr:`tp_clear` handler must be of the :c:type:`inquiry` type, or *NULL* +The :c:member:`~PyTypeObject.tp_clear` handler must be of the :c:type:`inquiry` type, or *NULL* if the object is immutable. diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 6c58c5d2f05..7a6b0a6aee2 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -427,6 +427,9 @@ pointer. standard :mod:`zlib` and :mod:`hashlib` modules release the GIL when compressing or hashing data. + +.. _gilstate: + Non-Python created threads -------------------------- @@ -905,41 +908,43 @@ Asynchronous Notifications A mechanism is provided to make asynchronous notifications to the main interpreter thread. These notifications take the form of a function -pointer and a void argument. +pointer and a void pointer argument. -.. index:: single: setcheckinterval() (in module sys) - -Every check interval, when the global interpreter lock is released and -reacquired, Python will also call any such provided functions. This can be used -for example by asynchronous IO handlers. The notification can be scheduled from -a worker thread and the actual call than made at the earliest convenience by the -main thread where it has possession of the global interpreter lock and can -perform any Python API calls. .. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) .. index:: single: Py_AddPendingCall() - Post a notification to the Python main thread. If successful, *func* will be - called with the argument *arg* at the earliest convenience. *func* will be - called having the global interpreter lock held and can thus use the full - Python API and can take any action such as setting object attributes to - signal IO completion. It must return 0 on success, or -1 signalling an - exception. The notification function won't be interrupted to perform another - asynchronous notification recursively, but it can still be interrupted to - switch threads if the global interpreter lock is released, for example, if it - calls back into Python code. + Schedule a function to be called from the main interpreter thread. On + success, 0 is returned and *func* is queued for being called in the + main thread. On failure, -1 is returned without setting any exception. - This function returns 0 on success in which case the notification has been - scheduled. Otherwise, for example if the notification buffer is full, it - returns -1 without setting any exception. + When successfully queued, *func* will be *eventually* called from the + main interpreter thread with the argument *arg*. It will be called + asynchronously with respect to normally running Python code, but with + both these conditions met: - This function can be called on any thread, be it a Python thread or some - other system thread. If it is a Python thread, it doesn't matter if it holds - the global interpreter lock or not. + * on a :term:`bytecode` boundary; + * with the main thread holding the :term:`global interpreter lock` + (*func* can therefore use the full C API). - .. versionadded:: 2.7 + *func* must return 0 on success, or -1 on failure with an exception + set. *func* won't be interrupted to perform another asynchronous + notification recursively, but it can still be interrupted to switch + threads if the global interpreter lock is released. + This function doesn't need a current thread state to run, and it doesn't + need the global interpreter lock. + + .. warning:: + This is a low-level function, only useful for very special cases. + There is no guarantee that *func* will be called as quick as + possible. If the main thread is busy executing a system call, + *func* won't be called before the system call returns. This + function is generally **not** suitable for calling Python code from + arbitrary C threads. Instead, use the :ref:`PyGILState API`. + + .. versionadded:: 2.7 .. _profiling: diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index c3a9ed633bd..6414277c849 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -255,8 +255,10 @@ sets all items of a list (actually, any mutable sequence) to a given item:: PyObject *index = PyInt_FromLong(i); if (!index) return -1; - if (PyObject_SetItem(target, index, item) < 0) + if (PyObject_SetItem(target, index, item) < 0) { + Py_DECREF(index); return -1; + } Py_DECREF(index); } return 0; diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst index 88ac0c1b2ff..8d1567c7b34 100644 --- a/Doc/c-api/iter.rst +++ b/Doc/c-api/iter.rst @@ -7,7 +7,7 @@ Iterator Protocol .. versionadded:: 2.2 -There are only a couple of functions specifically for working with iterators. +There are two functions specifically for working with iterators. .. c:function:: int PyIter_Check(PyObject *o) @@ -17,11 +17,10 @@ There are only a couple of functions specifically for working with iterators. .. c:function:: PyObject* PyIter_Next(PyObject *o) - Return the next value from the iteration *o*. If the object is an iterator, - this retrieves the next value from the iteration, and returns *NULL* with no - exception set if there are no remaining items. If the object is not an - iterator, :exc:`TypeError` is raised, or if there is an error in retrieving the - item, returns *NULL* and passes along the exception. + Return the next value from the iteration *o*. The object must be an iterator + (it is up to the caller to check this). If there are no remaining values, + returns *NULL* with no exception set. If an error occurs while retrieving + the item, returns *NULL* and passes along the exception. To write a loop which iterates over an iterator, the C code should look something like this:: diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index a02326f35e9..50b83e90d22 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -47,8 +47,8 @@ Object Protocol Generic attribute getter function that is meant to be put into a type object's ``tp_getattro`` slot. It looks for a descriptor in the dictionary of classes in the object's MRO as well as an attribute in the object's - :attr:`__dict__` (if present). As outlined in :ref:`descriptors`, data - descriptors take preference over instance attributes, while non-data + :attr:`~object.__dict__` (if present). As outlined in :ref:`descriptors`, + data descriptors take preference over instance attributes, while non-data descriptors don't. Otherwise, an :exc:`AttributeError` is raised. @@ -72,8 +72,8 @@ Object Protocol object's ``tp_setattro`` slot. It looks for a data descriptor in the dictionary of classes in the object's MRO, and if found it takes preference over setting the attribute in the instance dictionary. Otherwise, the - attribute is set in the object's :attr:`__dict__` (if present). Otherwise, - an :exc:`AttributeError` is raised and ``-1`` is returned. + attribute is set in the object's :attr:`~object.__dict__` (if present). + Otherwise, an :exc:`AttributeError` is raised and ``-1`` is returned. .. c:function:: int PyObject_DelAttr(PyObject *o, PyObject *attr_name) @@ -180,9 +180,9 @@ Object Protocol be done against every entry in *cls*. The result will be ``1`` when at least one of the checks returns ``1``, otherwise it will be ``0``. If *inst* is not a class instance and *cls* is neither a type object, nor a class object, nor a - tuple, *inst* must have a :attr:`__class__` attribute --- the class relationship - of the value of that attribute with *cls* will be used to determine the result - of this function. + tuple, *inst* must have a :attr:`~instance.__class__` attribute --- the + class relationship of the value of that attribute with *cls* will be used + to determine the result of this function. .. versionadded:: 2.1 @@ -196,9 +196,9 @@ of. If :class:`A` and :class:`B` are class objects, :class:`B` is a subclass of either is not a class object, a more general mechanism is used to determine the class relationship of the two objects. When testing if *B* is a subclass of *A*, if *A* is *B*, :c:func:`PyObject_IsSubclass` returns true. If *A* and *B* -are different objects, *B*'s :attr:`__bases__` attribute is searched in a -depth-first fashion for *A* --- the presence of the :attr:`__bases__` attribute -is considered sufficient for this determination. +are different objects, *B*'s :attr:`~class.__bases__` attribute is searched in +a depth-first fashion for *A* --- the presence of the :attr:`~class.__bases__` +attribute is considered sufficient for this determination. .. c:function:: int PyObject_IsSubclass(PyObject *derived, PyObject *cls) diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 41c4af4fee1..258530b1957 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -156,7 +156,7 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Return 1 if found and removed, 0 if not found (no action taken), and -1 if an error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a - :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`discard` + :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard` method, this function does not automatically convert unhashable sets into temporary frozensets. Raise :exc:`PyExc_SystemError` if *set* is an not an instance of :class:`set` or its subtype. diff --git a/Doc/c-api/string.rst b/Doc/c-api/string.rst index ecf70504bf0..32dc274a9e1 100644 --- a/Doc/c-api/string.rst +++ b/Doc/c-api/string.rst @@ -241,7 +241,7 @@ called with a non-string parameter. .. c:function:: PyObject* PyString_Format(PyObject *format, PyObject *args) Return a new string object from *format* and *args*. Analogous to ``format % - args``. The *args* argument must be a tuple. + args``. The *args* argument must be a tuple or dict. .. c:function:: void PyString_InternInPlace(PyObject **string) diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index f5007ace020..e31687f554d 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -293,6 +293,6 @@ definition with the same method name. .. c:function:: PyObject* Py_FindMethod(PyMethodDef table[], PyObject *ob, char *name) Return a bound method object for an extension type implemented in C. This - can be useful in the implementation of a :attr:`tp_getattro` or - :attr:`tp_getattr` handler that does not use the + can be useful in the implementation of a :c:member:`~PyTypeObject.tp_getattro` or + :c:member:`~PyTypeObject.tp_getattr` handler that does not use the :c:func:`PyObject_GenericGetAttr` function. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 7c37786ddf9..54f704a7d42 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -35,7 +35,7 @@ definition found there: The type object structure extends the :c:type:`PyVarObject` structure. The :attr:`ob_size` field is used for dynamic types (created by :func:`type_new`, usually called from a class statement). Note that :c:data:`PyType_Type` (the -metatype) initializes :attr:`tp_itemsize`, which means that its instances (i.e. +metatype) initializes :c:member:`~PyTypeObject.tp_itemsize`, which means that its instances (i.e. type objects) *must* have the :attr:`ob_size` field. @@ -108,7 +108,7 @@ type objects) *must* have the :attr:`ob_size` field. should be just the type name. If the module is a submodule of a package, the full package name is part of the full module name. For example, a type named :class:`T` defined in module :mod:`M` in subpackage :mod:`Q` in package :mod:`P` - should have the :attr:`tp_name` initializer ``"P.Q.M.T"``. + should have the :c:member:`~PyTypeObject.tp_name` initializer ``"P.Q.M.T"``. For dynamically allocated type objects, this should just be the type name, and the module name explicitly stored in the type dict as the value for key @@ -119,7 +119,7 @@ type objects) *must* have the :attr:`ob_size` field. attribute, and everything after the last dot is made accessible as the :attr:`__name__` attribute. - If no dot is present, the entire :attr:`tp_name` field is made accessible as the + If no dot is present, the entire :c:member:`~PyTypeObject.tp_name` field is made accessible as the :attr:`__name__` attribute, and the :attr:`__module__` attribute is undefined (unless explicitly set in the dictionary, as explained above). This means your type will be impossible to pickle. @@ -133,13 +133,13 @@ type objects) *must* have the :attr:`ob_size` field. These fields allow calculating the size in bytes of instances of the type. There are two kinds of types: types with fixed-length instances have a zero - :attr:`tp_itemsize` field, types with variable-length instances have a non-zero - :attr:`tp_itemsize` field. For a type with fixed-length instances, all - instances have the same size, given in :attr:`tp_basicsize`. + :c:member:`~PyTypeObject.tp_itemsize` field, types with variable-length instances have a non-zero + :c:member:`~PyTypeObject.tp_itemsize` field. For a type with fixed-length instances, all + instances have the same size, given in :c:member:`~PyTypeObject.tp_basicsize`. For a type with variable-length instances, the instances must have an - :attr:`ob_size` field, and the instance size is :attr:`tp_basicsize` plus N - times :attr:`tp_itemsize`, where N is the "length" of the object. The value of + :attr:`ob_size` field, and the instance size is :c:member:`~PyTypeObject.tp_basicsize` plus N + times :c:member:`~PyTypeObject.tp_itemsize`, where N is the "length" of the object. The value of N is typically stored in the instance's :attr:`ob_size` field. There are exceptions: for example, long ints use a negative :attr:`ob_size` to indicate a negative number, and N is ``abs(ob_size)`` there. Also, the presence of an @@ -152,21 +152,21 @@ type objects) *must* have the :attr:`ob_size` field. :c:macro:`PyObject_HEAD` or :c:macro:`PyObject_VAR_HEAD` (whichever is used to declare the instance struct) and this in turn includes the :attr:`_ob_prev` and :attr:`_ob_next` fields if they are present. This means that the only correct - way to get an initializer for the :attr:`tp_basicsize` is to use the + way to get an initializer for the :c:member:`~PyTypeObject.tp_basicsize` is to use the ``sizeof`` operator on the struct used to declare the instance layout. The basic size does not include the GC header size (this is new in Python 2.2; - in 2.1 and 2.0, the GC header size was included in :attr:`tp_basicsize`). + in 2.1 and 2.0, the GC header size was included in :c:member:`~PyTypeObject.tp_basicsize`). These fields are inherited separately by subtypes. If the base type has a - non-zero :attr:`tp_itemsize`, it is generally not safe to set - :attr:`tp_itemsize` to a different non-zero value in a subtype (though this + non-zero :c:member:`~PyTypeObject.tp_itemsize`, it is generally not safe to set + :c:member:`~PyTypeObject.tp_itemsize` to a different non-zero value in a subtype (though this depends on the implementation of the base type). A note about alignment: if the variable items require a particular alignment, - this should be taken care of by the value of :attr:`tp_basicsize`. Example: - suppose a type implements an array of ``double``. :attr:`tp_itemsize` is + this should be taken care of by the value of :c:member:`~PyTypeObject.tp_basicsize`. Example: + suppose a type implements an array of ``double``. :c:member:`~PyTypeObject.tp_itemsize` is ``sizeof(double)``. It is the programmer's responsibility that - :attr:`tp_basicsize` is a multiple of ``sizeof(double)`` (assuming this is the + :c:member:`~PyTypeObject.tp_basicsize` is a multiple of ``sizeof(double)`` (assuming this is the alignment requirement for ``double``). @@ -182,10 +182,10 @@ type objects) *must* have the :attr:`ob_size` field. destructor function should free all references which the instance owns, free all memory buffers owned by the instance (using the freeing function corresponding to the allocation function used to allocate the buffer), and finally (as its - last action) call the type's :attr:`tp_free` function. If the type is not + last action) call the type's :c:member:`~PyTypeObject.tp_free` function. If the type is not subtypable (doesn't have the :const:`Py_TPFLAGS_BASETYPE` flag bit set), it is permissible to call the object deallocator directly instead of via - :attr:`tp_free`. The object deallocator should be the one used to allocate the + :c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the instance; this is normally :c:func:`PyObject_Del` if the instance was allocated using :c:func:`PyObject_New` or :c:func:`PyObject_VarNew`, or :c:func:`PyObject_GC_Del` if the instance was allocated using @@ -199,26 +199,26 @@ type objects) *must* have the :attr:`ob_size` field. An optional pointer to the instance print function. The print function is only called when the instance is printed to a *real* file; - when it is printed to a pseudo-file (like a :class:`StringIO` instance), the - instance's :attr:`tp_repr` or :attr:`tp_str` function is called to convert it to - a string. These are also called when the type's :attr:`tp_print` field is - *NULL*. A type should never implement :attr:`tp_print` in a way that produces - different output than :attr:`tp_repr` or :attr:`tp_str` would. + when it is printed to a pseudo-file (like a :class:`~StringIO.StringIO` instance), the + instance's :c:member:`~PyTypeObject.tp_repr` or :c:member:`~PyTypeObject.tp_str` function is called to convert it to + a string. These are also called when the type's :c:member:`~PyTypeObject.tp_print` field is + *NULL*. A type should never implement :c:member:`~PyTypeObject.tp_print` in a way that produces + different output than :c:member:`~PyTypeObject.tp_repr` or :c:member:`~PyTypeObject.tp_str` would. The print function is called with the same signature as :c:func:`PyObject_Print`: ``int tp_print(PyObject *self, FILE *file, int flags)``. The *self* argument is the instance to be printed. The *file* argument is the stdio file to which it is to be printed. The *flags* argument is composed of flag bits. The only flag bit currently defined is :const:`Py_PRINT_RAW`. When the :const:`Py_PRINT_RAW` - flag bit is set, the instance should be printed the same way as :attr:`tp_str` + flag bit is set, the instance should be printed the same way as :c:member:`~PyTypeObject.tp_str` would format it; when the :const:`Py_PRINT_RAW` flag bit is clear, the instance - should be printed the same was as :attr:`tp_repr` would format it. It should + should be printed the same was as :c:member:`~PyTypeObject.tp_repr` would format it. It should return ``-1`` and set an exception condition when an error occurred during the comparison. - It is possible that the :attr:`tp_print` field will be deprecated. In any case, - it is recommended not to define :attr:`tp_print`, but instead to rely on - :attr:`tp_repr` and :attr:`tp_str` for printing. + It is possible that the :c:member:`~PyTypeObject.tp_print` field will be deprecated. In any case, + it is recommended not to define :c:member:`~PyTypeObject.tp_print`, but instead to rely on + :c:member:`~PyTypeObject.tp_repr` and :c:member:`~PyTypeObject.tp_str` for printing. This field is inherited by subtypes. @@ -228,13 +228,13 @@ type objects) *must* have the :attr:`ob_size` field. An optional pointer to the get-attribute-string function. This field is deprecated. When it is defined, it should point to a function - that acts the same as the :attr:`tp_getattro` function, but taking a C string + that acts the same as the :c:member:`~PyTypeObject.tp_getattro` function, but taking a C string instead of a Python string object to give the attribute name. The signature is the same as for :c:func:`PyObject_GetAttrString`. - This field is inherited by subtypes together with :attr:`tp_getattro`: a subtype - inherits both :attr:`tp_getattr` and :attr:`tp_getattro` from its base type when - the subtype's :attr:`tp_getattr` and :attr:`tp_getattro` are both *NULL*. + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattro`: a subtype + inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when + the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both *NULL*. .. c:member:: setattrfunc PyTypeObject.tp_setattr @@ -242,13 +242,13 @@ type objects) *must* have the :attr:`ob_size` field. An optional pointer to the set-attribute-string function. This field is deprecated. When it is defined, it should point to a function - that acts the same as the :attr:`tp_setattro` function, but taking a C string + that acts the same as the :c:member:`~PyTypeObject.tp_setattro` function, but taking a C string instead of a Python string object to give the attribute name. The signature is the same as for :c:func:`PyObject_SetAttrString`. - This field is inherited by subtypes together with :attr:`tp_setattro`: a subtype - inherits both :attr:`tp_setattr` and :attr:`tp_setattro` from its base type when - the subtype's :attr:`tp_setattr` and :attr:`tp_setattro` are both *NULL*. + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattro`: a subtype + inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when + the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both *NULL*. .. c:member:: cmpfunc PyTypeObject.tp_compare @@ -260,10 +260,10 @@ type objects) *must* have the :attr:`ob_size` field. *other*, and ``-1`` if *self* less than *other*. It should return ``-1`` and set an exception condition when an error occurred during the comparison. - This field is inherited by subtypes together with :attr:`tp_richcompare` and - :attr:`tp_hash`: a subtypes inherits all three of :attr:`tp_compare`, - :attr:`tp_richcompare`, and :attr:`tp_hash` when the subtype's - :attr:`tp_compare`, :attr:`tp_richcompare`, and :attr:`tp_hash` are all *NULL*. + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_richcompare` and + :c:member:`~PyTypeObject.tp_hash`: a subtypes inherits all three of :c:member:`~PyTypeObject.tp_compare`, + :c:member:`~PyTypeObject.tp_richcompare`, and :c:member:`~PyTypeObject.tp_hash` when the subtype's + :c:member:`~PyTypeObject.tp_compare`, :c:member:`~PyTypeObject.tp_richcompare`, and :c:member:`~PyTypeObject.tp_hash` are all *NULL*. .. c:member:: reprfunc PyTypeObject.tp_repr @@ -292,7 +292,7 @@ type objects) *must* have the :attr:`ob_size` field. objects which implement the number protocol. These fields are documented in :ref:`number-structs`. - The :attr:`tp_as_number` field is not inherited, but the contained fields are + The :c:member:`~PyTypeObject.tp_as_number` field is not inherited, but the contained fields are inherited individually. @@ -302,7 +302,7 @@ type objects) *must* have the :attr:`ob_size` field. objects which implement the sequence protocol. These fields are documented in :ref:`sequence-structs`. - The :attr:`tp_as_sequence` field is not inherited, but the contained fields + The :c:member:`~PyTypeObject.tp_as_sequence` field is not inherited, but the contained fields are inherited individually. @@ -312,7 +312,7 @@ type objects) *must* have the :attr:`ob_size` field. objects which implement the mapping protocol. These fields are documented in :ref:`mapping-structs`. - The :attr:`tp_as_mapping` field is not inherited, but the contained fields + The :c:member:`~PyTypeObject.tp_as_mapping` field is not inherited, but the contained fields are inherited individually. @@ -336,14 +336,14 @@ type objects) *must* have the :attr:`ob_size` field. the Python level will result in the ``tp_hash`` slot being set to :c:func:`PyObject_HashNotImplemented`. - When this field is not set, two possibilities exist: if the :attr:`tp_compare` - and :attr:`tp_richcompare` fields are both *NULL*, a default hash value based on + When this field is not set, two possibilities exist: if the :c:member:`~PyTypeObject.tp_compare` + and :c:member:`~PyTypeObject.tp_richcompare` fields are both *NULL*, a default hash value based on the object's address is returned; otherwise, a :exc:`TypeError` is raised. - This field is inherited by subtypes together with :attr:`tp_richcompare` and - :attr:`tp_compare`: a subtypes inherits all three of :attr:`tp_compare`, - :attr:`tp_richcompare`, and :attr:`tp_hash`, when the subtype's - :attr:`tp_compare`, :attr:`tp_richcompare` and :attr:`tp_hash` are all *NULL*. + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_richcompare` and + :c:member:`~PyTypeObject.tp_compare`: a subtypes inherits all three of :c:member:`~PyTypeObject.tp_compare`, + :c:member:`~PyTypeObject.tp_richcompare`, and :c:member:`~PyTypeObject.tp_hash`, when the subtype's + :c:member:`~PyTypeObject.tp_compare`, :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are all *NULL*. .. c:member:: ternaryfunc PyTypeObject.tp_call @@ -381,9 +381,9 @@ type objects) *must* have the :attr:`ob_size` field. convenient to set this field to :c:func:`PyObject_GenericGetAttr`, which implements the normal way of looking for object attributes. - This field is inherited by subtypes together with :attr:`tp_getattr`: a subtype - inherits both :attr:`tp_getattr` and :attr:`tp_getattro` from its base type when - the subtype's :attr:`tp_getattr` and :attr:`tp_getattro` are both *NULL*. + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattr`: a subtype + inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when + the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both *NULL*. .. c:member:: setattrofunc PyTypeObject.tp_setattro @@ -394,9 +394,9 @@ type objects) *must* have the :attr:`ob_size` field. convenient to set this field to :c:func:`PyObject_GenericSetAttr`, which implements the normal way of setting object attributes. - This field is inherited by subtypes together with :attr:`tp_setattr`: a subtype - inherits both :attr:`tp_setattr` and :attr:`tp_setattro` from its base type when - the subtype's :attr:`tp_setattr` and :attr:`tp_setattro` are both *NULL*. + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattr`: a subtype + inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when + the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both *NULL*. .. c:member:: PyBufferProcs* PyTypeObject.tp_as_buffer @@ -405,7 +405,7 @@ type objects) *must* have the :attr:`ob_size` field. which implement the buffer interface. These fields are documented in :ref:`buffer-structs`. - The :attr:`tp_as_buffer` field is not inherited, but the contained fields are + The :c:member:`~PyTypeObject.tp_as_buffer` field is not inherited, but the contained fields are inherited individually. @@ -414,8 +414,8 @@ type objects) *must* have the :attr:`ob_size` field. This field is a bit mask of various flags. Some flags indicate variant semantics for certain situations; others are used to indicate that certain fields in the type object (or in the extension structures referenced via - :attr:`tp_as_number`, :attr:`tp_as_sequence`, :attr:`tp_as_mapping`, and - :attr:`tp_as_buffer`) that were historically not always present are valid; if + :c:member:`~PyTypeObject.tp_as_number`, :c:member:`~PyTypeObject.tp_as_sequence`, :c:member:`~PyTypeObject.tp_as_mapping`, and + :c:member:`~PyTypeObject.tp_as_buffer`) that were historically not always present are valid; if such a flag bit is clear, the type fields it guards must not be accessed and must be considered to have a zero or *NULL* value instead. @@ -425,14 +425,14 @@ type objects) *must* have the :attr:`ob_size` field. inherited if the extension structure is inherited, i.e. the base type's value of the flag bit is copied into the subtype together with a pointer to the extension structure. The :const:`Py_TPFLAGS_HAVE_GC` flag bit is inherited together with - the :attr:`tp_traverse` and :attr:`tp_clear` fields, i.e. if the + the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields, i.e. if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the - :attr:`tp_traverse` and :attr:`tp_clear` fields in the subtype exist (as + :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist (as indicated by the :const:`Py_TPFLAGS_HAVE_RICHCOMPARE` flag bit) and have *NULL* values. The following bit masks are currently defined; these can be ORed together using - the ``|`` operator to form the value of the :attr:`tp_flags` field. The macro + the ``|`` operator to form the value of the :c:member:`~PyTypeObject.tp_flags` field. The macro :c:func:`PyType_HasFeature` takes a type and a flags value, *tp* and *f*, and checks whether ``tp->tp_flags & f`` is non-zero. @@ -440,13 +440,13 @@ type objects) *must* have the :attr:`ob_size` field. .. data:: Py_TPFLAGS_HAVE_GETCHARBUFFER If this bit is set, the :c:type:`PyBufferProcs` struct referenced by - :attr:`tp_as_buffer` has the :attr:`bf_getcharbuffer` field. + :c:member:`~PyTypeObject.tp_as_buffer` has the :attr:`bf_getcharbuffer` field. .. data:: Py_TPFLAGS_HAVE_SEQUENCE_IN If this bit is set, the :c:type:`PySequenceMethods` struct referenced by - :attr:`tp_as_sequence` has the :attr:`sq_contains` field. + :c:member:`~PyTypeObject.tp_as_sequence` has the :attr:`sq_contains` field. .. data:: Py_TPFLAGS_GC @@ -458,8 +458,8 @@ type objects) *must* have the :attr:`ob_size` field. .. data:: Py_TPFLAGS_HAVE_INPLACEOPS If this bit is set, the :c:type:`PySequenceMethods` struct referenced by - :attr:`tp_as_sequence` and the :c:type:`PyNumberMethods` structure referenced by - :attr:`tp_as_number` contain the fields for in-place operators. In particular, + :c:member:`~PyTypeObject.tp_as_sequence` and the :c:type:`PyNumberMethods` structure referenced by + :c:member:`~PyTypeObject.tp_as_number` contain the fields for in-place operators. In particular, this means that the :c:type:`PyNumberMethods` structure has the fields :attr:`nb_inplace_add`, :attr:`nb_inplace_subtract`, :attr:`nb_inplace_multiply`, :attr:`nb_inplace_divide`, @@ -473,7 +473,7 @@ type objects) *must* have the :attr:`ob_size` field. .. data:: Py_TPFLAGS_CHECKTYPES If this bit is set, the binary and ternary operations in the - :c:type:`PyNumberMethods` structure referenced by :attr:`tp_as_number` accept + :c:type:`PyNumberMethods` structure referenced by :c:member:`~PyTypeObject.tp_as_number` accept arguments of arbitrary object types, and do their own type conversions if needed. If this bit is clear, those operations require that all arguments have the current type as their type, and the caller is supposed to perform a coercion @@ -485,31 +485,31 @@ type objects) *must* have the :attr:`ob_size` field. .. data:: Py_TPFLAGS_HAVE_RICHCOMPARE - If this bit is set, the type object has the :attr:`tp_richcompare` field, as - well as the :attr:`tp_traverse` and the :attr:`tp_clear` fields. + If this bit is set, the type object has the :c:member:`~PyTypeObject.tp_richcompare` field, as + well as the :c:member:`~PyTypeObject.tp_traverse` and the :c:member:`~PyTypeObject.tp_clear` fields. .. data:: Py_TPFLAGS_HAVE_WEAKREFS - If this bit is set, the :attr:`tp_weaklistoffset` field is defined. Instances - of a type are weakly referenceable if the type's :attr:`tp_weaklistoffset` field + If this bit is set, the :c:member:`~PyTypeObject.tp_weaklistoffset` field is defined. Instances + of a type are weakly referenceable if the type's :c:member:`~PyTypeObject.tp_weaklistoffset` field has a value greater than zero. .. data:: Py_TPFLAGS_HAVE_ITER - If this bit is set, the type object has the :attr:`tp_iter` and - :attr:`tp_iternext` fields. + If this bit is set, the type object has the :c:member:`~PyTypeObject.tp_iter` and + :c:member:`~PyTypeObject.tp_iternext` fields. .. data:: Py_TPFLAGS_HAVE_CLASS If this bit is set, the type object has several new fields defined starting in - Python 2.2: :attr:`tp_methods`, :attr:`tp_members`, :attr:`tp_getset`, - :attr:`tp_base`, :attr:`tp_dict`, :attr:`tp_descr_get`, :attr:`tp_descr_set`, - :attr:`tp_dictoffset`, :attr:`tp_init`, :attr:`tp_alloc`, :attr:`tp_new`, - :attr:`tp_free`, :attr:`tp_is_gc`, :attr:`tp_bases`, :attr:`tp_mro`, - :attr:`tp_cache`, :attr:`tp_subclasses`, and :attr:`tp_weaklist`. + Python 2.2: :c:member:`~PyTypeObject.tp_methods`, :c:member:`~PyTypeObject.tp_members`, :c:member:`~PyTypeObject.tp_getset`, + :c:member:`~PyTypeObject.tp_base`, :c:member:`~PyTypeObject.tp_dict`, :c:member:`~PyTypeObject.tp_descr_get`, :c:member:`~PyTypeObject.tp_descr_set`, + :c:member:`~PyTypeObject.tp_dictoffset`, :c:member:`~PyTypeObject.tp_init`, :c:member:`~PyTypeObject.tp_alloc`, :c:member:`~PyTypeObject.tp_new`, + :c:member:`~PyTypeObject.tp_free`, :c:member:`~PyTypeObject.tp_is_gc`, :c:member:`~PyTypeObject.tp_bases`, :c:member:`~PyTypeObject.tp_mro`, + :c:member:`~PyTypeObject.tp_cache`, :c:member:`~PyTypeObject.tp_subclasses`, and :c:member:`~PyTypeObject.tp_weaklist`. .. data:: Py_TPFLAGS_HEAPTYPE @@ -547,7 +547,7 @@ type objects) *must* have the :attr:`ob_size` field. is set, instances must be created using :c:func:`PyObject_GC_New` and destroyed using :c:func:`PyObject_GC_Del`. More information in section :ref:`supporting-cycle-detection`. This bit also implies that the - GC-related fields :attr:`tp_traverse` and :attr:`tp_clear` are present in + GC-related fields :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` are present in the type object; but those fields also exist when :const:`Py_TPFLAGS_HAVE_GC` is clear but :const:`Py_TPFLAGS_HAVE_RICHCOMPARE` is set. @@ -582,8 +582,8 @@ The following three fields only exist if the about Python's garbage collection scheme can be found in section :ref:`supporting-cycle-detection`. - The :attr:`tp_traverse` pointer is used by the garbage collector to detect - reference cycles. A typical implementation of a :attr:`tp_traverse` function + The :c:member:`~PyTypeObject.tp_traverse` pointer is used by the garbage collector to detect + reference cycles. A typical implementation of a :c:member:`~PyTypeObject.tp_traverse` function simply calls :c:func:`Py_VISIT` on each of the instance's members that are Python objects. For example, this is function :c:func:`local_traverse` from the :mod:`thread` extension module:: @@ -603,15 +603,15 @@ The following three fields only exist if the On the other hand, even if you know a member can never be part of a cycle, as a debugging aid you may want to visit it anyway just so the :mod:`gc` module's - :func:`get_referents` function will include it. + :func:`~gc.get_referents` function will include it. Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to :c:func:`local_traverse` to have these specific names; don't name them just anything. - This field is inherited by subtypes together with :attr:`tp_clear` and the - :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :attr:`tp_traverse`, and - :attr:`tp_clear` are all inherited from the base type if they are all zero in + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_clear` and the + :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and + :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in the subtype *and* the subtype has the :const:`Py_TPFLAGS_HAVE_RICHCOMPARE` flag bit set. @@ -621,17 +621,17 @@ The following three fields only exist if the An optional pointer to a clear function for the garbage collector. This is only used if the :const:`Py_TPFLAGS_HAVE_GC` flag bit is set. - The :attr:`tp_clear` member function is used to break reference cycles in cyclic - garbage detected by the garbage collector. Taken together, all :attr:`tp_clear` + The :c:member:`~PyTypeObject.tp_clear` member function is used to break reference cycles in cyclic + garbage detected by the garbage collector. Taken together, all :c:member:`~PyTypeObject.tp_clear` functions in the system must combine to break all reference cycles. This is - subtle, and if in any doubt supply a :attr:`tp_clear` function. For example, - the tuple type does not implement a :attr:`tp_clear` function, because it's + subtle, and if in any doubt supply a :c:member:`~PyTypeObject.tp_clear` function. For example, + the tuple type does not implement a :c:member:`~PyTypeObject.tp_clear` function, because it's possible to prove that no reference cycle can be composed entirely of tuples. - Therefore the :attr:`tp_clear` functions of other types must be sufficient to + Therefore the :c:member:`~PyTypeObject.tp_clear` functions of other types must be sufficient to break any cycle containing a tuple. This isn't immediately obvious, and there's - rarely a good reason to avoid implementing :attr:`tp_clear`. + rarely a good reason to avoid implementing :c:member:`~PyTypeObject.tp_clear`. - Implementations of :attr:`tp_clear` should drop the instance's references to + Implementations of :c:member:`~PyTypeObject.tp_clear` should drop the instance's references to those of its members that may be Python objects, and set its pointers to those members to *NULL*, as in the following example:: @@ -656,18 +656,18 @@ The following three fields only exist if the so that *self* knows the contained object can no longer be used. The :c:func:`Py_CLEAR` macro performs the operations in a safe order. - Because the goal of :attr:`tp_clear` functions is to break reference cycles, + Because the goal of :c:member:`~PyTypeObject.tp_clear` functions is to break reference cycles, it's not necessary to clear contained objects like Python strings or Python integers, which can't participate in reference cycles. On the other hand, it may be convenient to clear all contained Python objects, and write the type's - :attr:`tp_dealloc` function to invoke :attr:`tp_clear`. + :c:member:`~PyTypeObject.tp_dealloc` function to invoke :c:member:`~PyTypeObject.tp_clear`. More information about Python's garbage collection scheme can be found in section :ref:`supporting-cycle-detection`. - This field is inherited by subtypes together with :attr:`tp_traverse` and the - :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :attr:`tp_traverse`, and - :attr:`tp_clear` are all inherited from the base type if they are all zero in + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_traverse` and the + :const:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and + :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in the subtype *and* the subtype has the :const:`Py_TPFLAGS_HAVE_RICHCOMPARE` flag bit set. @@ -688,13 +688,13 @@ The following three fields only exist if the comparisons makes sense (e.g. ``==`` and ``!=``, but not ``<`` and friends), directly raise :exc:`TypeError` in the rich comparison function. - This field is inherited by subtypes together with :attr:`tp_compare` and - :attr:`tp_hash`: a subtype inherits all three of :attr:`tp_compare`, - :attr:`tp_richcompare`, and :attr:`tp_hash`, when the subtype's - :attr:`tp_compare`, :attr:`tp_richcompare`, and :attr:`tp_hash` are all *NULL*. + This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_compare` and + :c:member:`~PyTypeObject.tp_hash`: a subtype inherits all three of :c:member:`~PyTypeObject.tp_compare`, + :c:member:`~PyTypeObject.tp_richcompare`, and :c:member:`~PyTypeObject.tp_hash`, when the subtype's + :c:member:`~PyTypeObject.tp_compare`, :c:member:`~PyTypeObject.tp_richcompare`, and :c:member:`~PyTypeObject.tp_hash` are all *NULL*. The following constants are defined to be used as the third argument for - :attr:`tp_richcompare` and for :c:func:`PyObject_RichCompare`: + :c:member:`~PyTypeObject.tp_richcompare` and for :c:func:`PyObject_RichCompare`: +----------------+------------+ | Constant | Comparison | @@ -725,26 +725,26 @@ set. instance structure needs to include a field of type :c:type:`PyObject\*` which is initialized to *NULL*. - Do not confuse this field with :attr:`tp_weaklist`; that is the list head for + Do not confuse this field with :c:member:`~PyTypeObject.tp_weaklist`; that is the list head for weak references to the type object itself. This field is inherited by subtypes, but see the rules listed below. A subtype may override this offset; this means that the subtype uses a different weak reference list head than the base type. Since the list head is always found via - :attr:`tp_weaklistoffset`, this should not be a problem. + :c:member:`~PyTypeObject.tp_weaklistoffset`, this should not be a problem. - When a type defined by a class statement has no :attr:`__slots__` declaration, + When a type defined by a class statement has no :attr:`~object.__slots__` declaration, and none of its base types are weakly referenceable, the type is made weakly referenceable by adding a weak reference list head slot to the instance layout - and setting the :attr:`tp_weaklistoffset` of that slot's offset. + and setting the :c:member:`~PyTypeObject.tp_weaklistoffset` of that slot's offset. When a type's :attr:`__slots__` declaration contains a slot named :attr:`__weakref__`, that slot becomes the weak reference list head for instances of the type, and the slot's offset is stored in the type's - :attr:`tp_weaklistoffset`. + :c:member:`~PyTypeObject.tp_weaklistoffset`. When a type's :attr:`__slots__` declaration does not contain a slot named - :attr:`__weakref__`, the type inherits its :attr:`tp_weaklistoffset` from its + :attr:`__weakref__`, the type inherits its :c:member:`~PyTypeObject.tp_weaklistoffset` from its base type. The next two fields only exist if the :const:`Py_TPFLAGS_HAVE_ITER` flag bit is @@ -772,7 +772,7 @@ set. are iterators (although classic instances always have this function, even if they don't define a :meth:`next` method). - Iterator types should also define the :attr:`tp_iter` function, and that + Iterator types should also define the :c:member:`~PyTypeObject.tp_iter` function, and that function should return the iterator instance itself (not a new iterator instance). @@ -780,7 +780,7 @@ set. This field is inherited by subtypes. -The next fields, up to and including :attr:`tp_weaklist`, only exist if the +The next fields, up to and including :c:member:`~PyTypeObject.tp_weaklist`, only exist if the :const:`Py_TPFLAGS_HAVE_CLASS` flag bit is set. @@ -790,7 +790,7 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the structures, declaring regular methods of this type. For each entry in the array, an entry is added to the type's dictionary (see - :attr:`tp_dict` below) containing a method descriptor. + :c:member:`~PyTypeObject.tp_dict` below) containing a method descriptor. This field is not inherited by subtypes (methods are inherited through a different mechanism). @@ -803,7 +803,7 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the this type. For each entry in the array, an entry is added to the type's dictionary (see - :attr:`tp_dict` below) containing a member descriptor. + :c:member:`~PyTypeObject.tp_dict` below) containing a member descriptor. This field is not inherited by subtypes (members are inherited through a different mechanism). @@ -815,7 +815,7 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the structures, declaring computed attributes of instances of this type. For each entry in the array, an entry is added to the type's dictionary (see - :attr:`tp_dict` below) containing a getset descriptor. + :c:member:`~PyTypeObject.tp_dict` below) containing a getset descriptor. This field is not inherited by subtypes (computed attributes are inherited through a different mechanism). @@ -894,7 +894,7 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the the instance variable dictionary; this offset is used by :c:func:`PyObject_GenericGetAttr`. - Do not confuse this field with :attr:`tp_dict`; that is the dictionary for + Do not confuse this field with :c:member:`~PyTypeObject.tp_dict`; that is the dictionary for attributes of the type object itself. If the value of this field is greater than zero, it specifies the offset from @@ -903,20 +903,20 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the offset is more expensive to use, and should only be used when the instance structure contains a variable-length part. This is used for example to add an instance variable dictionary to subtypes of :class:`str` or :class:`tuple`. Note - that the :attr:`tp_basicsize` field should account for the dictionary added to + that the :c:member:`~PyTypeObject.tp_basicsize` field should account for the dictionary added to the end in that case, even though the dictionary is not included in the basic object layout. On a system with a pointer size of 4 bytes, - :attr:`tp_dictoffset` should be set to ``-4`` to indicate that the dictionary is + :c:member:`~PyTypeObject.tp_dictoffset` should be set to ``-4`` to indicate that the dictionary is at the very end of the structure. The real dictionary offset in an instance can be computed from a negative - :attr:`tp_dictoffset` as follows:: + :c:member:`~PyTypeObject.tp_dictoffset` as follows:: dictoffset = tp_basicsize + abs(ob_size)*tp_itemsize + tp_dictoffset if dictoffset is not aligned on sizeof(void*): round up to sizeof(void*) - where :attr:`tp_basicsize`, :attr:`tp_itemsize` and :attr:`tp_dictoffset` are + where :c:member:`~PyTypeObject.tp_basicsize`, :c:member:`~PyTypeObject.tp_itemsize` and :c:member:`~PyTypeObject.tp_dictoffset` are taken from the type object, and :attr:`ob_size` is taken from the instance. The absolute value is taken because long ints use the sign of :attr:`ob_size` to store the sign of the number. (There's never a need to do this calculation @@ -925,17 +925,17 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the This field is inherited by subtypes, but see the rules listed below. A subtype may override this offset; this means that the subtype instances store the dictionary at a difference offset than the base type. Since the dictionary is - always found via :attr:`tp_dictoffset`, this should not be a problem. + always found via :c:member:`~PyTypeObject.tp_dictoffset`, this should not be a problem. - When a type defined by a class statement has no :attr:`__slots__` declaration, + When a type defined by a class statement has no :attr:`~object.__slots__` declaration, and none of its base types has an instance variable dictionary, a dictionary - slot is added to the instance layout and the :attr:`tp_dictoffset` is set to + slot is added to the instance layout and the :c:member:`~PyTypeObject.tp_dictoffset` is set to that slot's offset. When a type defined by a class statement has a :attr:`__slots__` declaration, - the type inherits its :attr:`tp_dictoffset` from its base type. + the type inherits its :c:member:`~PyTypeObject.tp_dictoffset` from its base type. - (Adding a slot named :attr:`__dict__` to the :attr:`__slots__` declaration does + (Adding a slot named :attr:`~object.__dict__` to the :attr:`__slots__` declaration does not have the expected effect, it just causes confusion. Maybe this should be added as a feature just like :attr:`__weakref__` though.) @@ -957,15 +957,15 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the arguments represent positional and keyword arguments of the call to :meth:`__init__`. - The :attr:`tp_init` function, if not *NULL*, is called when an instance is - created normally by calling its type, after the type's :attr:`tp_new` function - has returned an instance of the type. If the :attr:`tp_new` function returns an + The :c:member:`~PyTypeObject.tp_init` function, if not *NULL*, is called when an instance is + created normally by calling its type, after the type's :c:member:`~PyTypeObject.tp_new` function + has returned an instance of the type. If the :c:member:`~PyTypeObject.tp_new` function returns an instance of some other type that is not a subtype of the original type, no - :attr:`tp_init` function is called; if :attr:`tp_new` returns an instance of a - subtype of the original type, the subtype's :attr:`tp_init` is called. (VERSION + :c:member:`~PyTypeObject.tp_init` function is called; if :c:member:`~PyTypeObject.tp_new` returns an instance of a + subtype of the original type, the subtype's :c:member:`~PyTypeObject.tp_init` is called. (VERSION NOTE: described here is what is implemented in Python 2.2.1 and later. In - Python 2.2, the :attr:`tp_init` of the type of the object returned by - :attr:`tp_new` was always called, if not *NULL*.) + Python 2.2, the :c:member:`~PyTypeObject.tp_init` of the type of the object returned by + :c:member:`~PyTypeObject.tp_new` was always called, if not *NULL*.) This field is inherited by subtypes. @@ -982,14 +982,14 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the initialization. It should return a pointer to a block of memory of adequate length for the instance, suitably aligned, and initialized to zeros, but with :attr:`ob_refcnt` set to ``1`` and :attr:`ob_type` set to the type argument. If - the type's :attr:`tp_itemsize` is non-zero, the object's :attr:`ob_size` field + the type's :c:member:`~PyTypeObject.tp_itemsize` is non-zero, the object's :attr:`ob_size` field should be initialized to *nitems* and the length of the allocated memory block should be ``tp_basicsize + nitems*tp_itemsize``, rounded up to a multiple of ``sizeof(void*)``; otherwise, *nitems* is not used and the length of the block - should be :attr:`tp_basicsize`. + should be :c:member:`~PyTypeObject.tp_basicsize`. Do not use this function to do any other instance initialization, not even to - allocate additional memory; that should be done by :attr:`tp_new`. + allocate additional memory; that should be done by :c:member:`~PyTypeObject.tp_new`. This field is inherited by static subtypes, but not by dynamic subtypes (subtypes created by a class statement); in the latter, this field is always set @@ -1011,20 +1011,20 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the The subtype argument is the type of the object being created; the *args* and *kwds* arguments represent positional and keyword arguments of the call to the - type. Note that subtype doesn't have to equal the type whose :attr:`tp_new` + type. Note that subtype doesn't have to equal the type whose :c:member:`~PyTypeObject.tp_new` function is called; it may be a subtype of that type (but not an unrelated type). - The :attr:`tp_new` function should call ``subtype->tp_alloc(subtype, nitems)`` + The :c:member:`~PyTypeObject.tp_new` function should call ``subtype->tp_alloc(subtype, nitems)`` to allocate space for the object, and then do only as much further initialization as is absolutely necessary. Initialization that can safely be - ignored or repeated should be placed in the :attr:`tp_init` handler. A good + ignored or repeated should be placed in the :c:member:`~PyTypeObject.tp_init` handler. A good rule of thumb is that for immutable types, all initialization should take place - in :attr:`tp_new`, while for mutable types, most initialization should be - deferred to :attr:`tp_init`. + in :c:member:`~PyTypeObject.tp_new`, while for mutable types, most initialization should be + deferred to :c:member:`~PyTypeObject.tp_init`. This field is inherited by subtypes, except it is not inherited by static types - whose :attr:`tp_base` is *NULL* or ``&PyBaseObject_Type``. The latter exception + whose :c:member:`~PyTypeObject.tp_base` is *NULL* or ``&PyBaseObject_Type``. The latter exception is a precaution so that old extension types don't become callable simply by being linked with Python 2.2. @@ -1057,7 +1057,7 @@ The next fields, up to and including :attr:`tp_weaklist`, only exist if the The garbage collector needs to know whether a particular object is collectible or not. Normally, it is sufficient to look at the object's type's - :attr:`tp_flags` field, and check the :const:`Py_TPFLAGS_HAVE_GC` flag bit. But + :c:member:`~PyTypeObject.tp_flags` field, and check the :const:`Py_TPFLAGS_HAVE_GC` flag bit. But some types have a mixture of statically and dynamically allocated instances, and the statically allocated instances are not collectible. Such types should define this function; it should return ``1`` for a collectible instance, and @@ -1129,7 +1129,7 @@ subtypes. .. c:member:: PyTypeObject* PyTypeObject.tp_next - Pointer to the next type object with a non-zero :attr:`tp_allocs` field. + Pointer to the next type object with a non-zero :c:member:`~PyTypeObject.tp_allocs` field. Also, note that, in a garbage collected Python, tp_dealloc may be called from any Python thread, not just the thread which created the object (if the object @@ -1289,13 +1289,13 @@ Sequence Object Structures This function is used by :c:func:`PySequence_Concat` and has the same signature. It is also used by the ``+`` operator, after trying the numeric - addition via the :attr:`tp_as_number.nb_add` slot. + addition via the :c:member:`~PyTypeObject.tp_as_number.nb_add` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_repeat This function is used by :c:func:`PySequence_Repeat` and has the same signature. It is also used by the ``*`` operator, after trying numeric - multiplication via the :attr:`tp_as_number.nb_mul` slot. + multiplication via the :c:member:`~PyTypeObject.tp_as_number.nb_mul` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_item @@ -1348,14 +1348,14 @@ data as a set of chunks of data, where each chunk is specified as a pointer/length pair. These chunks are called :dfn:`segments` and are presumed to be non-contiguous in memory. -If an object does not export the buffer interface, then its :attr:`tp_as_buffer` +If an object does not export the buffer interface, then its :c:member:`~PyTypeObject.tp_as_buffer` member in the :c:type:`PyTypeObject` structure should be *NULL*. Otherwise, the -:attr:`tp_as_buffer` will point to a :c:type:`PyBufferProcs` structure. +:c:member:`~PyTypeObject.tp_as_buffer` will point to a :c:type:`PyBufferProcs` structure. .. note:: It is very important that your :c:type:`PyTypeObject` structure uses - :const:`Py_TPFLAGS_DEFAULT` for the value of the :attr:`tp_flags` member rather + :const:`Py_TPFLAGS_DEFAULT` for the value of the :c:member:`~PyTypeObject.tp_flags` member rather than ``0``. This tells the Python runtime that your :c:type:`PyBufferProcs` structure contains the :attr:`bf_getcharbuffer` slot. Older versions of Python did not have this member, so a new Python interpreter using an old extension @@ -1385,7 +1385,7 @@ member in the :c:type:`PyTypeObject` structure should be *NULL*. Otherwise, the The last slot is :attr:`bf_getcharbuffer`, of type :c:type:`getcharbufferproc`. This slot will only be present if the :const:`Py_TPFLAGS_HAVE_GETCHARBUFFER` - flag is present in the :attr:`tp_flags` field of the object's + flag is present in the :c:member:`~PyTypeObject.tp_flags` field of the object's :c:type:`PyTypeObject`. Before using this slot, the caller should test whether it is present by using the :c:func:`PyType_HasFeature` function. If the flag is present, :attr:`bf_getcharbuffer` may be *NULL*, indicating that the object's diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index fd4fe233f14..ddeaaa240b0 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -455,9 +455,9 @@ These are the generic codec APIs: Encode the :c:type:`Py_UNICODE` buffer *s* of the given *size* and return a Python string object. *encoding* and *errors* have the same meaning as the parameters - of the same name in the Unicode :meth:`encode` method. The codec to be used is - looked up using the Python codec registry. Return *NULL* if an exception was - raised by the codec. + of the same name in the Unicode :meth:`~unicode.encode` method. The codec + to be used is looked up using the Python codec registry. Return *NULL* if + an exception was raised by the codec. .. versionchanged:: 2.5 This function used an :c:type:`int` type for *size*. This might require diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 4ce3b03efbe..61076655774 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -265,7 +265,7 @@ the same library that the Python runtime is using. frame *f* is executed, interpreting bytecode and executing calls as needed. The additional *throwflag* parameter can mostly be ignored - if true, then it causes an exception to immediately be thrown; this is used for the - :meth:`throw` methods of generator objects. + :meth:`~generator.throw` methods of generator objects. .. c:function:: int PyEval_MergeCompilerFlags(PyCompilerFlags *cf) diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 1fc896fb73b..06c19d0412a 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -932,7 +932,7 @@ PyObject_CallMethod::...:: PyObject_CallMethodObjArgs:PyObject*::+1: PyObject_CallMethodObjArgs:PyObject*:o:0: -PyObject_CallMethodObjArgs:char*:name:: +PyObject_CallMethodObjArgs:PyObject*:name:0: PyObject_CallMethodObjArgs::...:: PyObject_CallObject:PyObject*::+1: diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst index f34a8eaa673..f121b54f485 100644 --- a/Doc/distutils/apiref.rst +++ b/Doc/distutils/apiref.rst @@ -729,7 +729,7 @@ This module provides the following functions. .. method:: CCompiler.execute(func, args[, msg=None, level=1]) - Invokes :func:`distutils.util.execute` This method invokes a Python function + Invokes :func:`distutils.util.execute`. This method invokes a Python function *func* with the given arguments *args*, after logging and taking into account the *dry_run* flag. @@ -989,8 +989,9 @@ directories. simply the list of all files under *src*, with the names changed to be under *dst*. - *preserve_mode* and *preserve_times* are the same as for :func:`copy_file` in - :mod:`distutils.file_util`; note that they only apply to regular files, not to + *preserve_mode* and *preserve_times* are the same as for + :func:`distutils.file_util.copy_file`; note that they only apply to + regular files, not to directories. If *preserve_symlinks* is true, symlinks will be copied as symlinks (on platforms that support them!); otherwise (the default), the destination of the symlink will be copied. *update* and *verbose* are the same @@ -1172,8 +1173,8 @@ other utility module. or :exc:`OSError`) exception object. Handles Python 1.5.1 and later styles, and does what it can to deal with exception objects that don't have a filename (which happens when the error is due to a two-file operation, such as - :func:`rename` or :func:`link`). Returns the error message as a string - prefixed with *prefix*. + :func:`~os.rename` or :func:`~os.link`). Returns the error message as a + string prefixed with *prefix*. .. function:: split_quoted(s) @@ -1257,8 +1258,8 @@ other utility module. built/installed/distributed -This module provides the :class:`Distribution` class, which represents the -module distribution being built/installed/distributed. +This module provides the :class:`~distutils.core.Distribution` class, which +represents the module distribution being built/installed/distributed. :mod:`distutils.extension` --- The Extension class @@ -1706,8 +1707,8 @@ This module supplies the abstract base class :class:`Command`. options, is the :meth:`run` method, which must also be implemented by every command class. - The class constructor takes a single argument *dist*, a :class:`Distribution` - instance. + The class constructor takes a single argument *dist*, a + :class:`~distutils.core.Distribution` instance. Creating a new Distutils command diff --git a/Doc/distutils/setupscript.rst b/Doc/distutils/setupscript.rst index 533af01f501..1d13a9cce94 100644 --- a/Doc/distutils/setupscript.rst +++ b/Doc/distutils/setupscript.rst @@ -139,7 +139,8 @@ directories, libraries to link with, etc.). All of this is done through another keyword argument to :func:`setup`, the :option:`ext_modules` option. :option:`ext_modules` is just a list of -:class:`Extension` instances, each of which describes a single extension module. +:class:`~distutils.core.Extension` instances, each of which describes a +single extension module. Suppose your distribution includes a single extension, called :mod:`foo` and implemented by :file:`foo.c`. If no additional instructions to the compiler/linker are needed, describing this extension is quite simple:: @@ -165,8 +166,8 @@ following sections. Extension names and packages ---------------------------- -The first argument to the :class:`Extension` constructor is always the name of -the extension, including any package names. For example, :: +The first argument to the :class:`~distutils.core.Extension` constructor is +always the name of the extension, including any package names. For example, :: Extension('foo', ['src/foo1.c', 'src/foo2.c']) @@ -196,7 +197,8 @@ will compile :file:`foo.c` to the extension :mod:`pkg.foo`, and :file:`bar.c` to Extension source files ---------------------- -The second argument to the :class:`Extension` constructor is a list of source +The second argument to the :class:`~distutils.core.Extension` constructor is +a list of source files. Since the Distutils currently only support C, C++, and Objective-C extensions, these are normally C/C++/Objective-C source files. (Be sure to use appropriate extensions to distinguish C++\ source files: :file:`.cc` and @@ -232,9 +234,9 @@ linked into the executable. Preprocessor options -------------------- -Three optional arguments to :class:`Extension` will help if you need to specify -include directories to search or preprocessor macros to define/undefine: -``include_dirs``, ``define_macros``, and ``undef_macros``. +Three optional arguments to :class:`~distutils.core.Extension` will help if +you need to specify include directories to search or preprocessor macros to +define/undefine: ``include_dirs``, ``define_macros``, and ``undef_macros``. For example, if your extension requires header files in the :file:`include` directory under your distribution root, use the ``include_dirs`` option:: diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst index f4d95b2b0e1..08b0cc2ceda 100644 --- a/Doc/extending/building.rst +++ b/Doc/extending/building.rst @@ -58,8 +58,9 @@ distutils; this section explains building extension modules only. It is common to pre-compute arguments to :func:`setup`, to better structure the driver script. In the example above, the\ ``ext_modules`` argument to :func:`setup` is a list of extension modules, each of which is an instance of -the :class:`Extension`. In the example, the instance defines an extension named -``demo`` which is build by compiling a single source file, :file:`demo.c`. +the :class:`~distutils.extension.Extension`. In the example, the instance +defines an extension named ``demo`` which is build by compiling a single source +file, :file:`demo.c`. In many cases, building an extension is more complex, since additional preprocessor defines and libraries may be needed. This is demonstrated in the diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index ae5efc40876..981e1d546e6 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -258,37 +258,55 @@ program. There is no need to recompile Python itself using C++. .. _link-reqs: -Linking Requirements -==================== - -While the :program:`configure` script shipped with the Python sources will -correctly build Python to export the symbols needed by dynamically linked -extensions, this is not automatically inherited by applications which embed the -Python library statically, at least on Unix. This is an issue when the -application is linked to the static runtime library (:file:`libpython.a`) and -needs to load dynamic extensions (implemented as :file:`.so` files). - -The problem is that some entry points are defined by the Python runtime solely -for extension modules to use. If the embedding application does not use any of -these entry points, some linkers will not include those entries in the symbol -table of the finished executable. Some additional options are needed to inform -the linker not to remove these symbols. - -Determining the right options to use for any given platform can be quite -difficult, but fortunately the Python configuration already has those values. -To retrieve them from an installed Python interpreter, start an interactive -interpreter and have a short session like this +Compiling and Linking under Unix-like systems +============================================= + +It is not necessarily trivial to find the right flags to pass to your +compiler (and linker) in order to embed the Python interpreter into your +application, particularly because Python needs to load library modules +implemented as C dynamic extensions (:file:`.so` files) linked against +it. + +To find out the required compiler and linker flags, you can execute the +:file:`python{X.Y}-config` script which is generated as part of the +installation process (a :file:`python-config` script may also be +available). This script has several options, of which the following will +be directly useful to you: + +* ``pythonX.Y-config --cflags`` will give you the recommended flags when + compiling:: + + $ /opt/bin/python2.7-config --cflags + -I/opt/include/python2.7 -fno-strict-aliasing -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes + +* ``pythonX.Y-config --ldflags`` will give you the recommended flags when + linking:: + + $ /opt/bin/python2.7-config --ldflags + -L/opt/lib/python2.7/config -lpthread -ldl -lutil -lm -lpython2.7 -Xlinker -export-dynamic + +.. note:: + To avoid confusion between several Python installations (and especially + between the system Python and your own compiled Python), it is recommended + that you use the absolute path to :file:`python{X.Y}-config`, as in the above + example. + +If this procedure doesn't work for you (it is not guaranteed to work for +all Unix-like platforms; however, we welcome :ref:`bug reports `) +you will have to read your system's documentation about dynamic linking and/or +examine Python's :file:`Makefile` (use :func:`sysconfig.get_makefile_filename` +to find its location) and compilation +options. In this case, the :mod:`sysconfig` module is a useful tool to +programmatically extract the configuration values that you will want to +combine together. For example: .. code-block:: python - >>> import distutils.sysconfig - >>> distutils.sysconfig.get_config_var('LINKFORSHARED') + >>> import sysconfig + >>> sysconfig.get_config_var('LIBS') + '-lpthread -ldl -lutil' + >>> sysconfig.get_config_var('LINKFORSHARED') '-Xlinker -export-dynamic' -.. index:: module: distutils.sysconfig - -The contents of the string presented will be the options that should be used. -If the string is empty, there's no need to add any additional options. The -:const:`LINKFORSHARED` definition corresponds to the variable of the same name -in Python's top-level :file:`Makefile`. +.. XXX similar documentation for Windows missing diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index eb18a46f871..8e8c3ab575b 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -372,6 +372,8 @@ to :c:func:`Py_Initialize`:: /* Add a static module */ initspam(); + ... + An example may be found in the file :file:`Demo/embed/demo.c` in the Python source distribution. @@ -510,7 +512,7 @@ or more format codes between parentheses. For example:: value of the Python function. :c:func:`PyObject_CallObject` is "reference-count-neutral" with respect to its arguments. In the example a new tuple was created to serve as the argument list, which is :c:func:`Py_DECREF`\ --ed immediately after the call. +-ed immediately after the :c:func:`PyObject_CallObject` call. The return value of :c:func:`PyObject_CallObject` is "new": either it is a brand new object, or it is an existing object whose reference count has been @@ -843,9 +845,9 @@ the cycle itself. The cycle detector is able to detect garbage cycles and can reclaim them so long as there are no finalizers implemented in Python (:meth:`__del__` methods). When there are such finalizers, the detector exposes the cycles through the -:mod:`gc` module (specifically, the -``garbage`` variable in that module). The :mod:`gc` module also exposes a way -to run the detector (the :func:`collect` function), as well as configuration +:mod:`gc` module (specifically, the :attr:`~gc.garbage` variable in that module). +The :mod:`gc` module also exposes a way to run the detector (the +:func:`~gc.collect` function), as well as configuration interfaces and the ability to disable the detector at runtime. The cycle detector is considered an optional component; though it is included by default, it can be disabled at build time using the :option:`--without-cycle-gc` option diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst index 75cf4c5d316..61129439958 100644 --- a/Doc/extending/index.rst +++ b/Doc/extending/index.rst @@ -5,12 +5,12 @@ ################################################## This document describes how to write modules in C or C++ to extend the Python -interpreter with new modules. Those modules can define new functions but also -new object types and their methods. The document also describes how to embed -the Python interpreter in another application, for use as an extension language. -Finally, it shows how to compile and link extension modules so that they can be -loaded dynamically (at run time) into the interpreter, if the underlying -operating system supports this feature. +interpreter with new modules. Those modules can not only define new functions +but also new object types and their methods. The document also describes how +to embed the Python interpreter in another application, for use as an extension +language. Finally, it shows how to compile and link extension modules so that +they can be loaded dynamically (at run time) into the interpreter, if the +underlying operating system supports this feature. This document assumes basic knowledge about Python. For an informal introduction to the language, see :ref:`tutorial-index`. :ref:`reference-index` diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 269c8fdd770..d76aa2499b2 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -150,11 +150,11 @@ This is so that Python knows how much memory to allocate when you call .. note:: If you want your type to be subclassable from Python, and your type has the same - :attr:`tp_basicsize` as its base type, you may have problems with multiple + :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple inheritance. A Python subclass of your type will have to list your type first - in its :attr:`__bases__`, or else it will not be able to call your type's + in its :attr:`~class.__bases__`, or else it will not be able to call your type's :meth:`__new__` method without getting an error. You can avoid this problem by - ensuring that your type has a larger value for :attr:`tp_basicsize` than its + ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its base type does. Most of the time, this will be true anyway, because either your base type will be :class:`object`, or else you will be adding data members to your base type, and therefore increasing its size. @@ -174,7 +174,7 @@ to :const:`Py_TPFLAGS_DEFAULT`. :: All types should include this constant in their flags. It enables all of the members defined by the current version of Python. -We provide a doc string for the type in :attr:`tp_doc`. :: +We provide a doc string for the type in :c:member:`~PyTypeObject.tp_doc`. :: "Noddy objects", /* tp_doc */ @@ -183,12 +183,12 @@ from the others. We aren't going to implement any of these in this version of the module. We'll expand this example later to have more interesting behavior. For now, all we want to be able to do is to create new :class:`Noddy` objects. -To enable object creation, we have to provide a :attr:`tp_new` implementation. +To enable object creation, we have to provide a :c:member:`~PyTypeObject.tp_new` implementation. In this case, we can just use the default implementation provided by the API function :c:func:`PyType_GenericNew`. We'd like to just assign this to the -:attr:`tp_new` slot, but we can't, for portability sake, On some platforms or +:c:member:`~PyTypeObject.tp_new` slot, but we can't, for portability sake, On some platforms or compilers, we can't statically initialize a structure member with a function -defined in another C module, so, instead, we'll assign the :attr:`tp_new` slot +defined in another C module, so, instead, we'll assign the :c:member:`~PyTypeObject.tp_new` slot in the module initialization function just before calling :c:func:`PyType_Ready`:: @@ -283,13 +283,13 @@ allocation and deallocation. At a minimum, we need a deallocation method:: self->ob_type->tp_free((PyObject*)self); } -which is assigned to the :attr:`tp_dealloc` member:: +which is assigned to the :c:member:`~PyTypeObject.tp_dealloc` member:: (destructor)Noddy_dealloc, /*tp_dealloc*/ This method decrements the reference counts of the two Python attributes. We use :c:func:`Py_XDECREF` here because the :attr:`first` and :attr:`last` members -could be *NULL*. It then calls the :attr:`tp_free` member of the object's type +could be *NULL*. It then calls the :c:member:`~PyTypeObject.tp_free` member of the object's type to free the object's memory. Note that the object's type might not be :class:`NoddyType`, because the object may be an instance of a subclass. @@ -323,7 +323,7 @@ strings, so we provide a new method:: return (PyObject *)self; } -and install it in the :attr:`tp_new` member:: +and install it in the :c:member:`~PyTypeObject.tp_new` member:: Noddy_new, /* tp_new */ @@ -344,16 +344,16 @@ created. New methods always accept positional and keyword arguments, but they often ignore the arguments, leaving the argument handling to initializer methods. Note that if the type supports subclassing, the type passed may not be the type being defined. The new method calls the tp_alloc slot to allocate -memory. We don't fill the :attr:`tp_alloc` slot ourselves. Rather +memory. We don't fill the :c:member:`~PyTypeObject.tp_alloc` slot ourselves. Rather :c:func:`PyType_Ready` fills it for us by inheriting it from our base class, which is :class:`object` by default. Most types use the default allocation. .. note:: - If you are creating a co-operative :attr:`tp_new` (one that calls a base type's - :attr:`tp_new` or :meth:`__new__`), you must *not* try to determine what method + If you are creating a co-operative :c:member:`~PyTypeObject.tp_new` (one that calls a base type's + :c:member:`~PyTypeObject.tp_new` or :meth:`__new__`), you must *not* try to determine what method to call using method resolution order at runtime. Always statically determine - what type you are going to call, and call its :attr:`tp_new` directly, or via + what type you are going to call, and call its :c:member:`~PyTypeObject.tp_new` directly, or via ``type->tp_base->tp_new``. If you do not do this, Python subclasses of your type that also inherit from other Python-defined classes may not work correctly. (Specifically, you may not be able to create instances of such subclasses @@ -390,11 +390,11 @@ We provide an initialization function:: return 0; } -by filling the :attr:`tp_init` slot. :: +by filling the :c:member:`~PyTypeObject.tp_init` slot. :: (initproc)Noddy_init, /* tp_init */ -The :attr:`tp_init` slot is exposed in Python as the :meth:`__init__` method. It +The :c:member:`~PyTypeObject.tp_init` slot is exposed in Python as the :meth:`__init__` method. It is used to initialize an object after it's created. Unlike the new method, we can't guarantee that the initializer is called. The initializer isn't called when unpickling objects and it can be overridden. Our initializer accepts @@ -424,7 +424,7 @@ reference counts. When don't we have to do this? * when we know that deallocation of the object [#]_ will not cause any calls back into our type's code -* when decrementing a reference count in a :attr:`tp_dealloc` handler when +* when decrementing a reference count in a :c:member:`~PyTypeObject.tp_dealloc` handler when garbage-collections is not supported [#]_ We want to expose our instance variables as attributes. There are a @@ -440,7 +440,7 @@ number of ways to do that. The simplest way is to define member definitions:: {NULL} /* Sentinel */ }; -and put the definitions in the :attr:`tp_members` slot:: +and put the definitions in the :c:member:`~PyTypeObject.tp_members` slot:: Noddy_members, /* tp_members */ @@ -516,7 +516,7 @@ definitions:: {NULL} /* Sentinel */ }; -and assign them to the :attr:`tp_methods` slot:: +and assign them to the :c:member:`~PyTypeObject.tp_methods` slot:: Noddy_methods, /* tp_methods */ @@ -611,7 +611,7 @@ We create an array of :c:type:`PyGetSetDef` structures:: {NULL} /* Sentinel */ }; -and register it in the :attr:`tp_getset` slot:: +and register it in the :c:member:`~PyTypeObject.tp_getset` slot:: Noddy_getseters, /* tp_getset */ @@ -628,7 +628,7 @@ We also remove the member definitions for these attributes:: {NULL} /* Sentinel */ }; -We also need to update the :attr:`tp_init` handler to only allow strings [#]_ to +We also need to update the :c:member:`~PyTypeObject.tp_init` handler to only allow strings [#]_ to be passed:: static int @@ -747,7 +747,7 @@ simplified:: .. note:: - Note that the :attr:`tp_traverse` implementation must name its arguments exactly + Note that the :c:member:`~PyTypeObject.tp_traverse` implementation must name its arguments exactly *visit* and *arg* in order to use :c:func:`Py_VISIT`. This is to encourage uniformity across these boring implementations. @@ -784,7 +784,7 @@ its reference count. We do this because, as was discussed earlier, if the reference count drops to zero, we might cause code to run that calls back into the object. In addition, because we now support garbage collection, we also have to worry about code being run that triggers garbage collection. If garbage -collection is run, our :attr:`tp_traverse` handler could get called. We can't +collection is run, our :c:member:`~PyTypeObject.tp_traverse` handler could get called. We can't take a chance of having :c:func:`Noddy_traverse` called when a member's reference count has dropped to zero and its value hasn't been set to *NULL*. @@ -804,8 +804,8 @@ Finally, we add the :const:`Py_TPFLAGS_HAVE_GC` flag to the class flags:: Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /*tp_flags*/ -That's pretty much it. If we had written custom :attr:`tp_alloc` or -:attr:`tp_free` slots, we'd need to modify them for cyclic-garbage collection. +That's pretty much it. If we had written custom :c:member:`~PyTypeObject.tp_alloc` or +:c:member:`~PyTypeObject.tp_free` slots, we'd need to modify them for cyclic-garbage collection. Most extensions will use the versions automatically provided. @@ -864,8 +864,8 @@ the :attr:`__init__` method of the base type. This pattern is important when writing a type with custom :attr:`new` and :attr:`dealloc` methods. The :attr:`new` method should not actually create the -memory for the object with :attr:`tp_alloc`, that will be handled by the base -class when calling its :attr:`tp_new`. +memory for the object with :c:member:`~PyTypeObject.tp_alloc`, that will be handled by the base +class when calling its :c:member:`~PyTypeObject.tp_new`. When filling out the :c:func:`PyTypeObject` for the :class:`Shoddy` type, you see a slot for :c:func:`tp_base`. Due to cross platform compiler issues, you can't @@ -890,8 +890,8 @@ the module's :c:func:`init` function. :: } Before calling :c:func:`PyType_Ready`, the type structure must have the -:attr:`tp_base` slot filled in. When we are deriving a new type, it is not -necessary to fill out the :attr:`tp_alloc` slot with :c:func:`PyType_GenericNew` +:c:member:`~PyTypeObject.tp_base` slot filled in. When we are deriving a new type, it is not +necessary to fill out the :c:member:`~PyTypeObject.tp_alloc` slot with :c:func:`PyType_GenericNew` -- the allocate function from the base type will be inherited. After that, calling :c:func:`PyType_Ready` and adding the type object to the @@ -934,7 +934,7 @@ that will be helpful in such a situation! :: These fields tell the runtime how much memory to allocate when new objects of this type are created. Python has some built-in support for variable length -structures (think: strings, lists) which is where the :attr:`tp_itemsize` field +structures (think: strings, lists) which is where the :c:member:`~PyTypeObject.tp_itemsize` field comes in. This will be dealt with later. :: char *tp_doc; @@ -1032,13 +1032,13 @@ that creating a temporary string object to be written to a file is too expensive. These handlers are all optional, and most types at most need to implement the -:attr:`tp_str` and :attr:`tp_repr` handlers. :: +:c:member:`~PyTypeObject.tp_str` and :c:member:`~PyTypeObject.tp_repr` handlers. :: reprfunc tp_repr; reprfunc tp_str; printfunc tp_print; -The :attr:`tp_repr` handler should return a string object containing a +The :c:member:`~PyTypeObject.tp_repr` handler should return a string object containing a representation of the instance for which it is called. Here is a simple example:: @@ -1049,15 +1049,15 @@ example:: obj->obj_UnderlyingDatatypePtr->size); } -If no :attr:`tp_repr` handler is specified, the interpreter will supply a -representation that uses the type's :attr:`tp_name` and a uniquely-identifying +If no :c:member:`~PyTypeObject.tp_repr` handler is specified, the interpreter will supply a +representation that uses the type's :c:member:`~PyTypeObject.tp_name` and a uniquely-identifying value for the object. -The :attr:`tp_str` handler is to :func:`str` what the :attr:`tp_repr` handler +The :c:member:`~PyTypeObject.tp_str` handler is to :func:`str` what the :c:member:`~PyTypeObject.tp_repr` handler described above is to :func:`repr`; that is, it is called when Python code calls :func:`str` on an instance of your object. Its implementation is very similar -to the :attr:`tp_repr` function, but the resulting string is intended for human -consumption. If :attr:`tp_str` is not specified, the :attr:`tp_repr` handler is +to the :c:member:`~PyTypeObject.tp_repr` function, but the resulting string is intended for human +consumption. If :c:member:`~PyTypeObject.tp_str` is not specified, the :c:member:`~PyTypeObject.tp_repr` handler is used instead. Here is a simple example:: @@ -1152,7 +1152,7 @@ type object to create :term:`descriptor`\s which are placed in the dictionary of type object. Each descriptor controls access to one attribute of the instance object. Each of the tables is optional; if all three are *NULL*, instances of the type will only have attributes that are inherited from their base type, and -should leave the :attr:`tp_getattro` and :attr:`tp_setattro` fields *NULL* as +should leave the :c:member:`~PyTypeObject.tp_getattro` and :c:member:`~PyTypeObject.tp_setattro` fields *NULL* as well, allowing the base type to handle attributes. The tables are declared as three fields of the type object:: @@ -1161,7 +1161,7 @@ The tables are declared as three fields of the type object:: struct PyMemberDef *tp_members; struct PyGetSetDef *tp_getset; -If :attr:`tp_methods` is not *NULL*, it must refer to an array of +If :c:member:`~PyTypeObject.tp_methods` is not *NULL*, it must refer to an array of :c:type:`PyMethodDef` structures. Each entry in the table is an instance of this structure:: @@ -1225,13 +1225,13 @@ combined using bitwise-OR. single: WRITE_RESTRICTED single: RESTRICTED -An interesting advantage of using the :attr:`tp_members` table to build +An interesting advantage of using the :c:member:`~PyTypeObject.tp_members` table to build descriptors that are used at runtime is that any attribute defined this way can have an associated doc string simply by providing the text in the table. An application can use the introspection API to retrieve the descriptor from the class object, and get the doc string using its :attr:`__doc__` attribute. -As with the :attr:`tp_methods` table, a sentinel entry with a :attr:`name` value +As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :attr:`name` value of *NULL* is required. .. XXX Descriptors need to be explained in more detail somewhere, but not here. @@ -1257,7 +1257,7 @@ portable to older versions of Python, and explains how the handler functions are called, so that if you do need to extend their functionality, you'll understand what needs to be done. -The :attr:`tp_getattr` handler is called when the object requires an attribute +The :c:member:`~PyTypeObject.tp_getattr` handler is called when the object requires an attribute look-up. It is called in the same situations where the :meth:`__getattr__` method of a class would be called. @@ -1265,7 +1265,7 @@ A likely way to handle this is (1) to implement a set of functions (such as :c:func:`newdatatype_getSize` and :c:func:`newdatatype_setSize` in the example below), (2) provide a method table listing these functions, and (3) provide a getattr function that returns the result of a lookup in that table. The method -table uses the same structure as the :attr:`tp_methods` field of the type +table uses the same structure as the :c:member:`~PyTypeObject.tp_methods` field of the type object. Here is an example:: @@ -1284,11 +1284,11 @@ Here is an example:: return Py_FindMethod(newdatatype_methods, (PyObject *)obj, name); } -The :attr:`tp_setattr` handler is called when the :meth:`__setattr__` or +The :c:member:`~PyTypeObject.tp_setattr` handler is called when the :meth:`__setattr__` or :meth:`__delattr__` method of a class instance would be called. When an attribute should be deleted, the third parameter will be *NULL*. Here is an example that simply raises an exception; if this were really all you wanted, the -:attr:`tp_setattr` handler should be set to *NULL*. :: +:c:member:`~PyTypeObject.tp_setattr` handler should be set to *NULL*. :: static int newdatatype_setattr(newdatatypeobject *obj, char *name, PyObject *v) @@ -1305,7 +1305,7 @@ Object Comparison cmpfunc tp_compare; -The :attr:`tp_compare` handler is called when comparisons are needed and the +The :c:member:`~PyTypeObject.tp_compare` handler is called when comparisons are needed and the object does not implement the specific rich comparison method which matches the requested comparison. (It is always used if defined and the :c:func:`PyObject_Compare` or :c:func:`PyObject_Cmp` functions are used, or if @@ -1316,7 +1316,7 @@ allowed to return arbitrary negative or positive integers for less than and greater than, respectively; as of Python 2.2, this is no longer allowed. In the future, other return values may be assigned a different meaning.) -A :attr:`tp_compare` handler may raise an exception. In this case it should +A :c:member:`~PyTypeObject.tp_compare` handler may raise an exception. In this case it should return a negative value. The caller has to test for the exception using :c:func:`PyErr_Occurred`. @@ -1391,7 +1391,7 @@ instance of your data type. Here is a moderately pointless example:: This function is called when an instance of your data type is "called", for example, if ``obj1`` is an instance of your data type and the Python script -contains ``obj1('hello')``, the :attr:`tp_call` handler is invoked. +contains ``obj1('hello')``, the :c:member:`~PyTypeObject.tp_call` handler is invoked. This function takes three arguments: @@ -1480,7 +1480,7 @@ those objects which do not benefit by weak referencing (such as numbers). For an object to be weakly referencable, the extension must include a :c:type:`PyObject\*` field in the instance structure for the use of the weak reference mechanism; it must be initialized to *NULL* by the object's -constructor. It must also set the :attr:`tp_weaklistoffset` field of the +constructor. It must also set the :c:member:`~PyTypeObject.tp_weaklistoffset` field of the corresponding type object to the offset of the field. For example, the instance type is defined with the following structure:: @@ -1566,7 +1566,7 @@ might be something like the following:: .. [#] This is true when we know that the object is a basic type, like a string or a float. -.. [#] We relied on this in the :attr:`tp_dealloc` handler in this example, because our +.. [#] We relied on this in the :c:member:`~PyTypeObject.tp_dealloc` handler in this example, because our type doesn't support garbage collection. Even if a type supports garbage collection, there are calls that can be made to "untrack" the object from garbage collection, however, these calls are advanced and not covered here. diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 833e58666b8..40babecdb70 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -372,20 +372,20 @@ Answer 2: Fortunately, there is `Stackless Python `_, which has a completely redesigned interpreter loop that avoids the C stack. -Why can't lambda forms contain statements? ------------------------------------------- +Why can't lambda expressions contain statements? +------------------------------------------------ -Python lambda forms cannot contain statements because Python's syntactic +Python lambda expressions cannot contain statements because Python's syntactic framework can't handle statements nested inside expressions. However, in Python, this is not a serious problem. Unlike lambda forms in other languages, where they add functionality, Python lambdas are only a shorthand notation if you're too lazy to define a function. Functions are already first class objects in Python, and can be declared in a -local scope. Therefore the only advantage of using a lambda form instead of a +local scope. Therefore the only advantage of using a lambda instead of a locally-defined function is that you don't need to invent a name for the function -- but that's just a local variable to which the function object (which -is exactly the same type of object that a lambda form yields) is assigned! +is exactly the same type of object that a lambda expression yields) is assigned! Can Python be compiled to machine code, C or some other language? @@ -910,8 +910,8 @@ There are several reasons to allow this. When you have a literal value for a list, tuple, or dictionary spread across multiple lines, it's easier to add more elements because you don't have to -remember to add a comma to the previous line. The lines can also be sorted in -your editor without creating a syntax error. +remember to add a comma to the previous line. The lines can also be reordered +without creating a syntax error. Accidentally omitting the comma can lead to errors that are hard to diagnose. For example:: diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index bb544202d9d..b34be2dcd6e 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -47,13 +47,15 @@ well as in freeware or shareware. Qt --- -There are bindings available for the Qt toolkit (`PyQt -`_) and for KDE (`PyKDE `__). If -you're writing open source software, you don't need to pay for PyQt, but if you -want to write proprietary applications, you must buy a PyQt license from -`Riverbank Computing `_ and (up to Qt 4.4; -Qt 4.5 upwards is licensed under the LGPL license) a Qt license from `Trolltech -`_. +There are bindings available for the Qt toolkit (using either `PyQt +`_ or `PySide +`_) and for KDE (`PyKDE `_). +PyQt is currently more mature than PySide, but you must buy a PyQt license from +`Riverbank Computing `_ +if you want to write proprietary applications. PySide is free for all applications. + +Qt 4.5 upwards is licensed under the LGPL license; also, commercial licenses +are available from `Nokia `_. Gtk+ ---- diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 2352db8e6ea..3878006a982 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -360,9 +360,9 @@ Why do lambdas defined in a loop with different values all return the same resul Assume you use a for loop to define a few different lambdas (or even plain functions), e.g.:: - squares = [] - for x in range(5): - squares.append(lambda: x**2) + >>> squares = [] + >>> for x in range(5): + ... squares.append(lambda: x**2) This gives you a list that contains 5 lambdas that calculate ``x**2``. You might expect that, when called, they would return, respectively, ``0``, ``1``, @@ -387,9 +387,9 @@ changing the value of ``x`` and see how the results of the lambdas change:: In order to avoid this, you need to save the values in variables local to the lambdas, so that they don't rely on the value of the global ``x``:: - squares = [] - for x in range(5): - squares.append(lambda n=x: n**2) + >>> squares = [] + >>> for x in range(5): + ... squares.append(lambda n=x: n**2) Here, ``n=x`` creates a new variable ``n`` local to the lambda and computed when the lambda is defined so that it has the same value that ``x`` had at @@ -748,11 +748,11 @@ Comma is not an operator in Python. Consider this session:: Since the comma is not an operator, but a separator between expressions the above is evaluated as if you had entered:: - >>> ("a" in "b"), "a" + ("a" in "b"), "a" not:: - >>> "a" in ("b", "a") + "a" in ("b", "a") The same is true of the various assignment operators (``=``, ``+=`` etc). They are not truly operators but syntactic delimiters in assignment statements. @@ -897,6 +897,7 @@ How do I modify a string in place? You can't, because strings are immutable. If you need an object with this ability, try converting the string to a list or use the array module:: + >>> import io >>> s = "Hello, world" >>> a = list(s) >>> print a @@ -910,7 +911,7 @@ ability, try converting the string to a list or use the array module:: >>> print a array('c', 'Hello, world') >>> a[0] = 'y' ; print a - array('c', 'yello world') + array('c', 'yello, world') >>> a.tostring() 'yello, world' @@ -1172,7 +1173,7 @@ How do I create a multidimensional list? You probably tried to make a multidimensional array like this:: - A = [[None] * 2] * 3 + >>> A = [[None] * 2] * 3 This looks correct if you print it:: @@ -1204,7 +1205,7 @@ use a list comprehension:: A = [[None] * w for i in range(h)] Or, you can use an extension that provides a matrix datatype; `Numeric Python -`_ is the best known. +`_ is the best known. How do I apply a method to a sequence of objects? @@ -1223,6 +1224,92 @@ More generically, you can try the following function:: return map(apply, methods, [arguments]*nobjects) +Why does a_tuple[i] += ['item'] raise an exception when the addition works? +--------------------------------------------------------------------------- + +This is because of a combination of the fact that augmented assignment +operators are *assignment* operators, and the difference between mutable and +immutable objects in Python. + +This discussion applies in general when augmented assignment operators are +applied to elements of a tuple that point to mutable objects, but we'll use +a ``list`` and ``+=`` as our exemplar. + +If you wrote:: + + >>> a_tuple = (1, 2) + >>> a_tuple[0] += 1 + Traceback (most recent call last): + ... + TypeError: 'tuple' object does not support item assignment + +The reason for the exception should be immediately clear: ``1`` is added to the +object ``a_tuple[0]`` points to (``1``), producing the result object, ``2``, +but when we attempt to assign the result of the computation, ``2``, to element +``0`` of the tuple, we get an error because we can't change what an element of +a tuple points to. + +Under the covers, what this augmented assignment statement is doing is +approximately this:: + + >>> result = a_tuple[0] + 1 + >>> a_tuple[0] = result + Traceback (most recent call last): + ... + TypeError: 'tuple' object does not support item assignment + +It is the assignment part of the operation that produces the error, since a +tuple is immutable. + +When you write something like:: + + >>> a_tuple = (['foo'], 'bar') + >>> a_tuple[0] += ['item'] + Traceback (most recent call last): + ... + TypeError: 'tuple' object does not support item assignment + +The exception is a bit more surprising, and even more surprising is the fact +that even though there was an error, the append worked:: + + >>> a_tuple[0] + ['foo', 'item'] + +To see why this happens, you need to know that (a) if an object implements an +``__iadd__`` magic method, it gets called when the ``+=`` augmented assignment +is executed, and its return value is what gets used in the assignment statement; +and (b) for lists, ``__iadd__`` is equivalent to calling ``extend`` on the list +and returning the list. That's why we say that for lists, ``+=`` is a +"shorthand" for ``list.extend``:: + + >>> a_list = [] + >>> a_list += [1] + >>> a_list + [1] + +This is equivalent to:: + + >>> result = a_list.__iadd__([1]) + >>> a_list = result + +The object pointed to by a_list has been mutated, and the pointer to the +mutated object is assigned back to ``a_list``. The end result of the +assignment is a no-op, since it is a pointer to the same object that ``a_list`` +was previously pointing to, but the assignment still happens. + +Thus, in our tuple example what is happening is equivalent to:: + + >>> result = a_tuple[0].__iadd__(['item']) + >>> a_tuple[0] = result + Traceback (most recent call last): + ... + TypeError: 'tuple' object does not support item assignment + +The ``__iadd__`` succeeds, and thus the list is extended, but even though +``result`` points to the same object that ``a_tuple[0]`` already points to, +that final assignment still results in an error, because tuples are immutable. + + Dictionaries ============ @@ -1654,13 +1741,13 @@ file is automatic if you're importing a module and Python has the ability (permissions, free space, etc...) to write the compiled module back to the directory. -Running Python on a top level script is not considered an import and no ``.pyc`` -will be created. For example, if you have a top-level module ``abc.py`` that -imports another module ``xyz.py``, when you run abc, ``xyz.pyc`` will be created -since xyz is imported, but no ``abc.pyc`` file will be created since ``abc.py`` -isn't being imported. +Running Python on a top level script is not considered an import and no +``.pyc`` will be created. For example, if you have a top-level module +``foo.py`` that imports another module ``xyz.py``, when you run ``foo``, +``xyz.pyc`` will be created since ``xyz`` is imported, but no ``foo.pyc`` file +will be created since ``foo.py`` isn't being imported. -If you need to create abc.pyc -- that is, to create a .pyc file for a module +If you need to create ``foo.pyc`` -- that is, to create a ``.pyc`` file for a module that is not imported -- you can, using the :mod:`py_compile` and :mod:`compileall` modules. @@ -1668,9 +1755,9 @@ The :mod:`py_compile` module can manually compile any module. One way is to use the ``compile()`` function in that module interactively:: >>> import py_compile - >>> py_compile.compile('abc.py') + >>> py_compile.compile('foo.py') # doctest: +SKIP -This will write the ``.pyc`` to the same location as ``abc.py`` (or you can +This will write the ``.pyc`` to the same location as ``foo.py`` (or you can override that with the optional parameter ``cfile``). You can also automatically compile all files in a directory or directories using diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 3641e5c189d..b5e81717832 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -77,6 +77,14 @@ Glossary Benevolent Dictator For Life, a.k.a. `Guido van Rossum `_, Python's creator. + bytes-like object + An object that supports the :ref:`buffer protocol `, + like :class:`str`, :class:`bytearray` or :class:`memoryview`. + Bytes-like objects can be used for various operations that expect + binary data, such as compression, saving to a binary file or sending + over a socket. Some operations need the binary data to be mutable, + in which case not all bytes-like objects can apply. + bytecode Python source code is compiled into bytecode, the internal representation of a Python program in the CPython interpreter. The bytecode is also @@ -357,6 +365,10 @@ Glossary fraction. Integer division can be forced by using the ``//`` operator instead of the ``/`` operator. See also :term:`__future__`. + importing + The process by which Python code in one module is made available to + Python code in another module. + importer An object that both finds and loads a module; both a :term:`finder` and :term:`loader` object. @@ -501,6 +513,13 @@ Glossary for a member during lookup. See `The Python 2.3 Method Resolution Order `_. + module + An object that serves as an organizational unit of Python code. Modules + have a namespace containing arbitrary Python objects. Modules are loaded + into Python by the process of :term:`importing`. + + See also :term:`package`. + MRO See :term:`method resolution order`. @@ -544,7 +563,7 @@ Glossary new-style class Any class which inherits from :class:`object`. This includes all built-in types like :class:`list` and :class:`dict`. Only new-style classes can - use Python's newer, versatile features like :attr:`__slots__`, + use Python's newer, versatile features like :attr:`~object.__slots__`, descriptors, properties, and :meth:`__getattribute__`. More information can be found in :ref:`newstyle`. @@ -554,6 +573,11 @@ Glossary (methods). Also the ultimate base class of any :term:`new-style class`. + package + A Python :term:`module` which can contain submodules or recursively, + subpackages. Technically, a package is a Python module with an + ``__path__`` attribute. + parameter A named entity in a :term:`function` (or method) definition that specifies an :term:`argument` (or in some cases, arguments) that the @@ -656,7 +680,7 @@ Glossary statement A statement is part of a suite (a "block" of code). A statement is either - an :term:`expression` or a one of several constructs with a keyword, such + an :term:`expression` or one of several constructs with a keyword, such as :keyword:`if`, :keyword:`while` or :keyword:`for`. struct sequence @@ -679,7 +703,8 @@ Glossary type The type of a Python object determines what kind of object it is; every object has a type. An object's type is accessible as its - :attr:`__class__` attribute or can be retrieved with ``type(obj)``. + :attr:`~instance.__class__` attribute or can be retrieved with + ``type(obj)``. universal newlines A manner of interpreting text streams in which all of the following are diff --git a/Doc/howto/advocacy.rst b/Doc/howto/advocacy.rst deleted file mode 100644 index 2969d266adf..00000000000 --- a/Doc/howto/advocacy.rst +++ /dev/null @@ -1,355 +0,0 @@ -************************* - Python Advocacy HOWTO -************************* - -:Author: A.M. Kuchling -:Release: 0.03 - - -.. topic:: Abstract - - It's usually difficult to get your management to accept open source software, - and Python is no exception to this rule. This document discusses reasons to use - Python, strategies for winning acceptance, facts and arguments you can use, and - cases where you *shouldn't* try to use Python. - - -Reasons to Use Python -===================== - -There are several reasons to incorporate a scripting language into your -development process, and this section will discuss them, and why Python has some -properties that make it a particularly good choice. - - -Programmability ---------------- - -Programs are often organized in a modular fashion. Lower-level operations are -grouped together, and called by higher-level functions, which may in turn be -used as basic operations by still further upper levels. - -For example, the lowest level might define a very low-level set of functions for -accessing a hash table. The next level might use hash tables to store the -headers of a mail message, mapping a header name like ``Date`` to a value such -as ``Tue, 13 May 1997 20:00:54 -0400``. A yet higher level may operate on -message objects, without knowing or caring that message headers are stored in a -hash table, and so forth. - -Often, the lowest levels do very simple things; they implement a data structure -such as a binary tree or hash table, or they perform some simple computation, -such as converting a date string to a number. The higher levels then contain -logic connecting these primitive operations. Using the approach, the primitives -can be seen as basic building blocks which are then glued together to produce -the complete product. - -Why is this design approach relevant to Python? Because Python is well suited -to functioning as such a glue language. A common approach is to write a Python -module that implements the lower level operations; for the sake of speed, the -implementation might be in C, Java, or even Fortran. Once the primitives are -available to Python programs, the logic underlying higher level operations is -written in the form of Python code. The high-level logic is then more -understandable, and easier to modify. - -John Ousterhout wrote a paper that explains this idea at greater length, -entitled "Scripting: Higher Level Programming for the 21st Century". I -recommend that you read this paper; see the references for the URL. Ousterhout -is the inventor of the Tcl language, and therefore argues that Tcl should be -used for this purpose; he only briefly refers to other languages such as Python, -Perl, and Lisp/Scheme, but in reality, Ousterhout's argument applies to -scripting languages in general, since you could equally write extensions for any -of the languages mentioned above. - - -Prototyping ------------ - -In *The Mythical Man-Month*, Fredrick Brooks suggests the following rule when -planning software projects: "Plan to throw one away; you will anyway." Brooks -is saying that the first attempt at a software design often turns out to be -wrong; unless the problem is very simple or you're an extremely good designer, -you'll find that new requirements and features become apparent once development -has actually started. If these new requirements can't be cleanly incorporated -into the program's structure, you're presented with two unpleasant choices: -hammer the new features into the program somehow, or scrap everything and write -a new version of the program, taking the new features into account from the -beginning. - -Python provides you with a good environment for quickly developing an initial -prototype. That lets you get the overall program structure and logic right, and -you can fine-tune small details in the fast development cycle that Python -provides. Once you're satisfied with the GUI interface or program output, you -can translate the Python code into C++, Fortran, Java, or some other compiled -language. - -Prototyping means you have to be careful not to use too many Python features -that are hard to implement in your other language. Using ``eval()``, or regular -expressions, or the :mod:`pickle` module, means that you're going to need C or -Java libraries for formula evaluation, regular expressions, and serialization, -for example. But it's not hard to avoid such tricky code, and in the end the -translation usually isn't very difficult. The resulting code can be rapidly -debugged, because any serious logical errors will have been removed from the -prototype, leaving only more minor slip-ups in the translation to track down. - -This strategy builds on the earlier discussion of programmability. Using Python -as glue to connect lower-level components has obvious relevance for constructing -prototype systems. In this way Python can help you with development, even if -end users never come in contact with Python code at all. If the performance of -the Python version is adequate and corporate politics allow it, you may not need -to do a translation into C or Java, but it can still be faster to develop a -prototype and then translate it, instead of attempting to produce the final -version immediately. - -One example of this development strategy is Microsoft Merchant Server. Version -1.0 was written in pure Python, by a company that subsequently was purchased by -Microsoft. Version 2.0 began to translate the code into C++, shipping with some -C++code and some Python code. Version 3.0 didn't contain any Python at all; all -the code had been translated into C++. Even though the product doesn't contain -a Python interpreter, the Python language has still served a useful purpose by -speeding up development. - -This is a very common use for Python. Past conference papers have also -described this approach for developing high-level numerical algorithms; see -David M. Beazley and Peter S. Lomdahl's paper "Feeding a Large-scale Physics -Application to Python" in the references for a good example. If an algorithm's -basic operations are things like "Take the inverse of this 4000x4000 matrix", -and are implemented in some lower-level language, then Python has almost no -additional performance cost; the extra time required for Python to evaluate an -expression like ``m.invert()`` is dwarfed by the cost of the actual computation. -It's particularly good for applications where seemingly endless tweaking is -required to get things right. GUI interfaces and Web sites are prime examples. - -The Python code is also shorter and faster to write (once you're familiar with -Python), so it's easier to throw it away if you decide your approach was wrong; -if you'd spent two weeks working on it instead of just two hours, you might -waste time trying to patch up what you've got out of a natural reluctance to -admit that those two weeks were wasted. Truthfully, those two weeks haven't -been wasted, since you've learnt something about the problem and the technology -you're using to solve it, but it's human nature to view this as a failure of -some sort. - - -Simplicity and Ease of Understanding ------------------------------------- - -Python is definitely *not* a toy language that's only usable for small tasks. -The language features are general and powerful enough to enable it to be used -for many different purposes. It's useful at the small end, for 10- or 20-line -scripts, but it also scales up to larger systems that contain thousands of lines -of code. - -However, this expressiveness doesn't come at the cost of an obscure or tricky -syntax. While Python has some dark corners that can lead to obscure code, there -are relatively few such corners, and proper design can isolate their use to only -a few classes or modules. It's certainly possible to write confusing code by -using too many features with too little concern for clarity, but most Python -code can look a lot like a slightly-formalized version of human-understandable -pseudocode. - -In *The New Hacker's Dictionary*, Eric S. Raymond gives the following definition -for "compact": - -.. epigraph:: - - Compact *adj.* Of a design, describes the valuable property that it can all be - apprehended at once in one's head. This generally means the thing created from - the design can be used with greater facility and fewer errors than an equivalent - tool that is not compact. Compactness does not imply triviality or lack of - power; for example, C is compact and FORTRAN is not, but C is more powerful than - FORTRAN. Designs become non-compact through accreting features and cruft that - don't merge cleanly into the overall design scheme (thus, some fans of Classic C - maintain that ANSI C is no longer compact). - - (From http://www.catb.org/~esr/jargon/html/C/compact.html) - -In this sense of the word, Python is quite compact, because the language has -just a few ideas, which are used in lots of places. Take namespaces, for -example. Import a module with ``import math``, and you create a new namespace -called ``math``. Classes are also namespaces that share many of the properties -of modules, and have a few of their own; for example, you can create instances -of a class. Instances? They're yet another namespace. Namespaces are currently -implemented as Python dictionaries, so they have the same methods as the -standard dictionary data type: .keys() returns all the keys, and so forth. - -This simplicity arises from Python's development history. The language syntax -derives from different sources; ABC, a relatively obscure teaching language, is -one primary influence, and Modula-3 is another. (For more information about ABC -and Modula-3, consult their respective Web sites at http://www.cwi.nl/~steven/abc/ -and http://www.m3.org.) Other features have come from C, Icon, -Algol-68, and even Perl. Python hasn't really innovated very much, but instead -has tried to keep the language small and easy to learn, building on ideas that -have been tried in other languages and found useful. - -Simplicity is a virtue that should not be underestimated. It lets you learn the -language more quickly, and then rapidly write code -- code that often works the -first time you run it. - - -Java Integration ----------------- - -If you're working with Java, Jython (http://www.jython.org/) is definitely worth -your attention. Jython is a re-implementation of Python in Java that compiles -Python code into Java bytecodes. The resulting environment has very tight, -almost seamless, integration with Java. It's trivial to access Java classes -from Python, and you can write Python classes that subclass Java classes. -Jython can be used for prototyping Java applications in much the same way -CPython is used, and it can also be used for test suites for Java code, or -embedded in a Java application to add scripting capabilities. - - -Arguments and Rebuttals -======================= - -Let's say that you've decided upon Python as the best choice for your -application. How can you convince your management, or your fellow developers, -to use Python? This section lists some common arguments against using Python, -and provides some possible rebuttals. - -**Python is freely available software that doesn't cost anything. How good can -it be?** - -Very good, indeed. These days Linux and Apache, two other pieces of open source -software, are becoming more respected as alternatives to commercial software, -but Python hasn't had all the publicity. - -Python has been around for several years, with many users and developers. -Accordingly, the interpreter has been used by many people, and has gotten most -of the bugs shaken out of it. While bugs are still discovered at intervals, -they're usually either quite obscure (they'd have to be, for no one to have run -into them before) or they involve interfaces to external libraries. The -internals of the language itself are quite stable. - -Having the source code should be viewed as making the software available for -peer review; people can examine the code, suggest (and implement) improvements, -and track down bugs. To find out more about the idea of open source code, along -with arguments and case studies supporting it, go to http://www.opensource.org. - -**Who's going to support it?** - -Python has a sizable community of developers, and the number is still growing. -The Internet community surrounding the language is an active one, and is worth -being considered another one of Python's advantages. Most questions posted to -the comp.lang.python newsgroup are quickly answered by someone. - -Should you need to dig into the source code, you'll find it's clear and -well-organized, so it's not very difficult to write extensions and track down -bugs yourself. If you'd prefer to pay for support, there are companies and -individuals who offer commercial support for Python. - -**Who uses Python for serious work?** - -Lots of people; one interesting thing about Python is the surprising diversity -of applications that it's been used for. People are using Python to: - -* Run Web sites - -* Write GUI interfaces - -* Control number-crunching code on supercomputers - -* Make a commercial application scriptable by embedding the Python interpreter - inside it - -* Process large XML data sets - -* Build test suites for C or Java code - -Whatever your application domain is, there's probably someone who's used Python -for something similar. Yet, despite being useable for such high-end -applications, Python's still simple enough to use for little jobs. - -See http://wiki.python.org/moin/OrganizationsUsingPython for a list of some of -the organizations that use Python. - -**What are the restrictions on Python's use?** - -They're practically nonexistent. Consult :ref:`history-and-license` for the full -language, but it boils down to three conditions: - -* You have to leave the copyright notice on the software; if you don't include - the source code in a product, you have to put the copyright notice in the - supporting documentation. - -* Don't claim that the institutions that have developed Python endorse your - product in any way. - -* If something goes wrong, you can't sue for damages. Practically all software - licenses contain this condition. - -Notice that you don't have to provide source code for anything that contains -Python or is built with it. Also, the Python interpreter and accompanying -documentation can be modified and redistributed in any way you like, and you -don't have to pay anyone any licensing fees at all. - -**Why should we use an obscure language like Python instead of well-known -language X?** - -I hope this HOWTO, and the documents listed in the final section, will help -convince you that Python isn't obscure, and has a healthily growing user base. -One word of advice: always present Python's positive advantages, instead of -concentrating on language X's failings. People want to know why a solution is -good, rather than why all the other solutions are bad. So instead of attacking -a competing solution on various grounds, simply show how Python's virtues can -help. - - -Useful Resources -================ - -http://www.pythonology.com/success - The Python Success Stories are a collection of stories from successful users of - Python, with the emphasis on business and corporate users. - -.. http://www.fsbassociates.com/books/pythonchpt1.htm - The first chapter of \emph{Internet Programming with Python} also - examines some of the reasons for using Python. The book is well worth - buying, but the publishers have made the first chapter available on - the Web. - -http://www.tcl.tk/doc/scripting.html - John Ousterhout's white paper on scripting is a good argument for the utility of - scripting languages, though naturally enough, he emphasizes Tcl, the language he - developed. Most of the arguments would apply to any scripting language. - -http://www.python.org/workshops/1997-10/proceedings/beazley.html - The authors, David M. Beazley and Peter S. Lomdahl, describe their use of - Python at Los Alamos National Laboratory. It's another good example of how - Python can help get real work done. This quotation from the paper has been - echoed by many people: - - .. epigraph:: - - Originally developed as a large monolithic application for massively parallel - processing systems, we have used Python to transform our application into a - flexible, highly modular, and extremely powerful system for performing - simulation, data analysis, and visualization. In addition, we describe how - Python has solved a number of important problems related to the development, - debugging, deployment, and maintenance of scientific software. - -http://pythonjournal.cognizor.com/pyj1/Everitt-Feit_interview98-V1.html - This interview with Andy Feit, discussing Infoseek's use of Python, can be used - to show that choosing Python didn't introduce any difficulties into a company's - development process, and provided some substantial benefits. - -.. http://www.python.org/psa/Commercial.html - Robin Friedrich wrote this document on how to support Python's use in - commercial projects. - -http://www.python.org/workshops/1997-10/proceedings/stein.ps - For the 6th Python conference, Greg Stein presented a paper that traced Python's - adoption and usage at a startup called eShop, and later at Microsoft. - -http://www.opensource.org - Management may be doubtful of the reliability and usefulness of software that - wasn't written commercially. This site presents arguments that show how open - source software can have considerable advantages over closed-source software. - -http://www.faqs.org/docs/Linux-mini/Advocacy.html - The Linux Advocacy mini-HOWTO was the inspiration for this document, and is also - well worth reading for general suggestions on winning acceptance for a new - technology, such as Linux or Python. In general, you won't make much progress - by simply attacking existing systems and complaining about their inadequacies; - this often ends up looking like unfocused whining. It's much better to point - out some of the many areas where Python is an improvement over other systems. - diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 8cc638eb61c..f110a55bba7 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -11,7 +11,7 @@ recommended command-line parsing module in the Python standard library. .. note:: - There's two other modules that fulfill the same task, namely + There are two other modules that fulfill the same task, namely :mod:`getopt` (an equivalent for :c:func:`getopt` from the C language) and the deprecated :mod:`optparse`. Note also that :mod:`argparse` is based on :mod:`optparse`, @@ -63,7 +63,7 @@ A few concepts we can learn from the four commands: * That's a snippet of the help text. It's very useful in that you can come across a program you have never used before, and can figure out - how it works simply by reading it's help text. + how it works simply by reading its help text. The basics @@ -468,7 +468,7 @@ verbosity argument (check the output of ``python --help``):: print answer We have introduced another action, "count", -to count the number of occurences of a specific optional arguments: +to count the number of occurrences of a specific optional arguments: .. code-block:: sh @@ -668,8 +668,8 @@ Conflicting options So far, we have been working with two methods of an :class:`argparse.ArgumentParser` instance. Let's introduce a third one, :meth:`add_mutually_exclusive_group`. It allows for us to specify options that -conflict with each other. Let's also change the rest of the program make the -new functionality makes more sense: +conflict with each other. Let's also change the rest of the program so that +the new functionality makes more sense: we'll introduce the ``--quiet`` option, which will be the opposite of the ``--verbose`` one:: diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index 1fc10c763da..74c1f2a4402 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -144,8 +144,8 @@ window, but you might wish to divide the screen into smaller windows, in order to redraw or clear them separately. The :func:`newwin` function creates a new window of a given size, returning the new window object. :: - begin_x = 20 ; begin_y = 7 - height = 5 ; width = 40 + begin_x = 20; begin_y = 7 + height = 5; width = 40 win = curses.newwin(height, width, begin_y, begin_x) A word about the coordinate system used in curses: coordinates are always passed @@ -184,11 +184,13 @@ displayed. :: # explained in the next section for y in range(0, 100): for x in range(0, 100): - try: pad.addch(y,x, ord('a') + (x*x+y*y) % 26 ) - except curses.error: pass + try: + pad.addch(y,x, ord('a') + (x*x+y*y) % 26) + except curses.error: + pass # Displays a section of the pad in the middle of the screen - pad.refresh( 0,0, 5,5, 20,75) + pad.refresh(0,0, 5,5, 20,75) The :func:`refresh` call displays a section of the pad in the rectangle extending from coordinate (5,5) to coordinate (20,75) on the screen; the upper @@ -321,7 +323,7 @@ again, such combinations are not guaranteed to work on all terminals. An example, which displays a line of text using color pair 1:: - stdscr.addstr( "Pretty text", curses.color_pair(1) ) + stdscr.addstr("Pretty text", curses.color_pair(1)) stdscr.refresh() As I said before, a color pair consists of a foreground and background color. @@ -343,7 +345,7 @@ When you change a color pair, any text already displayed using that color pair will change to the new colors. You can also display new text in this color with:: - stdscr.addstr(0,0, "RED ALERT!", curses.color_pair(1) ) + stdscr.addstr(0,0, "RED ALERT!", curses.color_pair(1)) Very fancy terminals can change the definitions of the actual colors to a given RGB value. This lets you change color 1, which is usually red, to purple or @@ -381,9 +383,12 @@ your program will look something like this:: while 1: c = stdscr.getch() - if c == ord('p'): PrintDocument() - elif c == ord('q'): break # Exit the while() - elif c == curses.KEY_HOME: x = y = 0 + if c == ord('p'): + PrintDocument() + elif c == ord('q'): + break # Exit the while() + elif c == curses.KEY_HOME: + x = y = 0 The :mod:`curses.ascii` module supplies ASCII class membership functions that take either integer or 1-character-string arguments; these may be useful in @@ -433,4 +438,3 @@ If you write an interesting little program, feel free to contribute it as another demo. We can always use more of them! The ncurses FAQ: http://invisible-island.net/ncurses/ncurses.faq.html - diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index 6706f894d12..e4c95b1f520 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -13,7 +13,6 @@ Currently, the HOWTOs are: .. toctree:: :maxdepth: 1 - advocacy.rst pyporting.rst cporting.rst curses.rst diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 422bf624c74..e5d75a64b9b 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -97,11 +97,11 @@ The output looks like this:: Multiple handlers and formatters -------------------------------- -Loggers are plain Python objects. The :func:`addHandler` method has no minimum -or maximum quota for the number of handlers you may add. Sometimes it will be -beneficial for an application to log all messages of all severities to a text -file while simultaneously logging errors or above to the console. To set this -up, simply configure the appropriate handlers. The logging calls in the +Loggers are plain Python objects. The :meth:`~Logger.addHandler` method has no +minimum or maximum quota for the number of handlers you may add. Sometimes it +will be beneficial for an application to log all messages of all severities to a +text file while simultaneously logging errors or above to the console. To set +this up, simply configure the appropriate handlers. The logging calls in the application code will remain unchanged. Here is a slight modification to the previous simple module-based configuration example:: @@ -395,8 +395,9 @@ printed on the console; on the server side, you should see something like:: Note that there are some security issues with pickle in some scenarios. If these affect you, you can use an alternative serialization scheme by overriding -the :meth:`makePickle` method and implementing your alternative there, as -well as adapting the above script to use your alternative serialization. +the :meth:`~handlers.SocketHandler.makePickle` method and implementing your +alternative there, as well as adapting the above script to use your alternative +serialization. .. _context-info: @@ -404,6 +405,8 @@ well as adapting the above script to use your alternative serialization. Adding contextual information to your logging output ---------------------------------------------------- +.. currentmodule:: logging + Sometimes you want logging output to contain contextual information in addition to the parameters passed to the logging call. For example, in a networked application, it may be desirable to log client-specific information @@ -445,9 +448,9 @@ information in the delegated call. Here's a snippet from the code of msg, kwargs = self.process(msg, kwargs) self.logger.debug(msg, *args, **kwargs) -The :meth:`process` method of :class:`LoggerAdapter` is where the contextual -information is added to the logging output. It's passed the message and -keyword arguments of the logging call, and it passes back (potentially) +The :meth:`~LoggerAdapter.process` method of :class:`LoggerAdapter` is where the +contextual information is added to the logging output. It's passed the message +and keyword arguments of the logging call, and it passes back (potentially) modified versions of these to use in the call to the underlying logger. The default implementation of this method leaves the message alone, but inserts an 'extra' key in the keyword argument whose value is the dict-like object @@ -459,70 +462,32 @@ merged into the :class:`LogRecord` instance's __dict__, allowing you to use customized strings with your :class:`Formatter` instances which know about the keys of the dict-like object. If you need a different method, e.g. if you want to prepend or append the contextual information to the message string, -you just need to subclass :class:`LoggerAdapter` and override :meth:`process` -to do what you need. Here's an example script which uses this class, which -also illustrates what dict-like behaviour is needed from an arbitrary -'dict-like' object for use in the constructor:: - - import logging +you just need to subclass :class:`LoggerAdapter` and override +:meth:`~LoggerAdapter.process` to do what you need. Here is a simple example:: - class ConnInfo: - """ - An example class which shows how an arbitrary class can be used as - the 'extra' context information repository passed to a LoggerAdapter. - """ + class CustomAdapter(logging.LoggerAdapter): + """ + This example adapter expects the passed in dict-like object to have a + 'connid' key, whose value in brackets is prepended to the log message. + """ + def process(self, msg, kwargs): + return '[%s] %s' % (self.extra['connid'], msg), kwargs - def __getitem__(self, name): - """ - To allow this instance to look like a dict. - """ - from random import choice - if name == 'ip': - result = choice(['127.0.0.1', '192.168.0.1']) - elif name == 'user': - result = choice(['jim', 'fred', 'sheila']) - else: - result = self.__dict__.get(name, '?') - return result +which you can use like this:: - def __iter__(self): - """ - To allow iteration over keys, which will be merged into - the LogRecord dict before formatting and output. - """ - keys = ['ip', 'user'] - keys.extend(self.__dict__.keys()) - return keys.__iter__() + logger = logging.getLogger(__name__) + adapter = CustomAdapter(logger, {'connid': some_conn_id}) - if __name__ == '__main__': - from random import choice - levels = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL) - a1 = logging.LoggerAdapter(logging.getLogger('a.b.c'), - { 'ip' : '123.231.231.123', 'user' : 'sheila' }) - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)-15s %(name)-5s %(levelname)-8s IP: %(ip)-15s User: %(user)-8s %(message)s') - a1.debug('A debug message') - a1.info('An info message with %s', 'some parameters') - a2 = logging.LoggerAdapter(logging.getLogger('d.e.f'), ConnInfo()) - for x in range(10): - lvl = choice(levels) - lvlname = logging.getLevelName(lvl) - a2.log(lvl, 'A message at %s level with %d %s', lvlname, 2, 'parameters') +Then any events that you log to the adapter will have the value of +``some_conn_id`` prepended to the log messages. -When this script is run, the output should look something like this:: +Using objects other than dicts to pass contextual information +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - 2008-01-18 14:49:54,023 a.b.c DEBUG IP: 123.231.231.123 User: sheila A debug message - 2008-01-18 14:49:54,023 a.b.c INFO IP: 123.231.231.123 User: sheila An info message with some parameters - 2008-01-18 14:49:54,023 d.e.f CRITICAL IP: 192.168.0.1 User: jim A message at CRITICAL level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f INFO IP: 192.168.0.1 User: jim A message at INFO level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f WARNING IP: 192.168.0.1 User: sheila A message at WARNING level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f ERROR IP: 127.0.0.1 User: fred A message at ERROR level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f ERROR IP: 127.0.0.1 User: sheila A message at ERROR level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f WARNING IP: 192.168.0.1 User: sheila A message at WARNING level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f WARNING IP: 192.168.0.1 User: jim A message at WARNING level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f INFO IP: 192.168.0.1 User: fred A message at INFO level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f WARNING IP: 192.168.0.1 User: sheila A message at WARNING level with 2 parameters - 2008-01-18 14:49:54,033 d.e.f WARNING IP: 127.0.0.1 User: jim A message at WARNING level with 2 parameters +You don't need to pass an actual dict to a :class:`LoggerAdapter` - you could +pass an instance of a class which implements ``__getitem__`` and ``__iter__`` so +that it looks like a dict to logging. This would be useful if you want to +generate values dynamically (whereas the values in a dict would be constant). .. _filters-contextual: @@ -607,25 +572,23 @@ threads in a single process *is* supported, logging to a single file from *multiple processes* is *not* supported, because there is no standard way to serialize access to a single file across multiple processes in Python. If you need to log to a single file from multiple processes, one way of doing this is -to have all the processes log to a :class:`SocketHandler`, and have a separate -process which implements a socket server which reads from the socket and logs -to file. (If you prefer, you can dedicate one thread in one of the existing -processes to perform this function.) :ref:`This section ` -documents this approach in more detail and includes a working socket receiver -which can be used as a starting point for you to adapt in your own -applications. +to have all the processes log to a :class:`~handlers.SocketHandler`, and have a +separate process which implements a socket server which reads from the socket +and logs to file. (If you prefer, you can dedicate one thread in one of the +existing processes to perform this function.) +:ref:`This section ` documents this approach in more detail and +includes a working socket receiver which can be used as a starting point for you +to adapt in your own applications. If you are using a recent version of Python which includes the :mod:`multiprocessing` module, you could write your own handler which uses the -:class:`Lock` class from this module to serialize access to the file from -your processes. The existing :class:`FileHandler` and subclasses do not make -use of :mod:`multiprocessing` at present, though they may do so in the future. -Note that at present, the :mod:`multiprocessing` module does not provide +:class:`~multiprocessing.Lock` class from this module to serialize access to the +file from your processes. The existing :class:`FileHandler` and subclasses do +not make use of :mod:`multiprocessing` at present, though they may do so in the +future. Note that at present, the :mod:`multiprocessing` module does not provide working lock functionality on all platforms (see http://bugs.python.org/issue3770). -.. currentmodule:: logging.handlers - Using file rotation ------------------- @@ -637,7 +600,7 @@ Sometimes you want to let a log file grow to a certain size, then open a new file and log to that. You may want to keep a certain number of these files, and when that many files have been created, rotate the files so that the number of files and the size of the files both remain bounded. For this usage pattern, the -logging package provides a :class:`RotatingFileHandler`:: +logging package provides a :class:`~handlers.RotatingFileHandler`:: import glob import logging @@ -688,7 +651,7 @@ An example dictionary-based configuration Below is an example of a logging configuration dictionary - it's taken from the `documentation on the Django project `_. -This dictionary is passed to :func:`~logging.config.dictConfig` to put the configuration into effect:: +This dictionary is passed to :func:`~config.dictConfig` to put the configuration into effect:: LOGGING = { 'version': 1, diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index f6e7153a5cb..1916c47bb7b 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -469,12 +469,13 @@ Handlers :class:`~logging.Handler` objects are responsible for dispatching the appropriate log messages (based on the log messages' severity) to the handler's -specified destination. Logger objects can add zero or more handler objects to -themselves with an :func:`addHandler` method. As an example scenario, an -application may want to send all log messages to a log file, all log messages -of error or higher to stdout, and all messages of critical to an email address. -This scenario requires three individual handlers where each handler is -responsible for sending messages of a specific severity to a specific location. +specified destination. :class:`Logger` objects can add zero or more handler +objects to themselves with an :meth:`~Logger.addHandler` method. As an example +scenario, an application may want to send all log messages to a log file, all +log messages of error or higher to stdout, and all messages of critical to an +email address. This scenario requires three individual handlers where each +handler is responsible for sending messages of a specific severity to a specific +location. The standard library includes quite a few handler types (see :ref:`useful-handlers`); the tutorials use mainly :class:`StreamHandler` and @@ -485,16 +486,17 @@ themselves with. The only handler methods that seem relevant for application developers who are using the built-in handler objects (that is, not creating custom handlers) are the following configuration methods: -* The :meth:`Handler.setLevel` method, just as in logger objects, specifies the +* The :meth:`~Handler.setLevel` method, just as in logger objects, specifies the lowest severity that will be dispatched to the appropriate destination. Why are there two :func:`setLevel` methods? The level set in the logger determines which severity of messages it will pass to its handlers. The level set in each handler determines which messages that handler will send on. -* :func:`setFormatter` selects a Formatter object for this handler to use. +* :meth:`~Handler.setFormatter` selects a Formatter object for this handler to + use. -* :func:`addFilter` and :func:`removeFilter` respectively configure and - deconfigure filter objects on handlers. +* :meth:`~Handler.addFilter` and :meth:`~Handler.removeFilter` respectively + configure and deconfigure filter objects on handlers. Application code should not directly instantiate and use instances of :class:`Handler`. Instead, the :class:`Handler` class is a base class that @@ -918,16 +920,16 @@ Logged messages are formatted for presentation through instances of the use with the % operator and a dictionary. For formatting multiple messages in a batch, instances of -:class:`BufferingFormatter` can be used. In addition to the format string (which -is applied to each message in the batch), there is provision for header and -trailer format strings. +:class:`~handlers.BufferingFormatter` can be used. In addition to the format +string (which is applied to each message in the batch), there is provision for +header and trailer format strings. When filtering based on logger level and/or handler level is not enough, instances of :class:`Filter` can be added to both :class:`Logger` and -:class:`Handler` instances (through their :meth:`addFilter` method). Before -deciding to process a message further, both loggers and handlers consult all -their filters for permission. If any filter returns a false value, the message -is not processed further. +:class:`Handler` instances (through their :meth:`~Handler.addFilter` method). +Before deciding to process a message further, both loggers and handlers consult +all their filters for permission. If any filter returns a false value, the +message is not processed further. The basic :class:`Filter` functionality allows filtering by specific logger name. If this feature is used, messages sent to the named logger and its @@ -945,19 +947,20 @@ in production. This is so that errors which occur while handling logging events cause the application using logging to terminate prematurely. :class:`SystemExit` and :class:`KeyboardInterrupt` exceptions are never -swallowed. Other exceptions which occur during the :meth:`emit` method of a -:class:`Handler` subclass are passed to its :meth:`handleError` method. +swallowed. Other exceptions which occur during the :meth:`~Handler.emit` method +of a :class:`Handler` subclass are passed to its :meth:`~Handler.handleError` +method. -The default implementation of :meth:`handleError` in :class:`Handler` checks -to see if a module-level variable, :data:`raiseExceptions`, is set. If set, a -traceback is printed to :data:`sys.stderr`. If not set, the exception is swallowed. +The default implementation of :meth:`~Handler.handleError` in :class:`Handler` +checks to see if a module-level variable, :data:`raiseExceptions`, is set. If +set, a traceback is printed to :data:`sys.stderr`. If not set, the exception is +swallowed. .. note:: The default value of :data:`raiseExceptions` is ``True``. This is because during development, you typically want to be notified of any exceptions that occur. It's advised that you set :data:`raiseExceptions` to ``False`` for production usage. -.. currentmodule:: logging .. _arbitrary-object-messages: @@ -967,11 +970,11 @@ Using arbitrary objects as messages In the preceding sections and examples, it has been assumed that the message passed when logging the event is a string. However, this is not the only possibility. You can pass an arbitrary object as a message, and its -:meth:`__str__` method will be called when the logging system needs to convert -it to a string representation. In fact, if you want to, you can avoid +:meth:`~object.__str__` method will be called when the logging system needs to +convert it to a string representation. In fact, if you want to, you can avoid computing a string representation altogether - for example, the -:class:`SocketHandler` emits an event by pickling it and sending it over the -wire. +:class:`~handlers.SocketHandler` emits an event by pickling it and sending it +over the wire. Optimization @@ -980,9 +983,10 @@ Optimization Formatting of message arguments is deferred until it cannot be avoided. However, computing the arguments passed to the logging method can also be expensive, and you may want to avoid doing it if the logger will just throw -away your event. To decide what to do, you can call the :meth:`isEnabledFor` -method which takes a level argument and returns true if the event would be -created by the Logger for that level of call. You can write code like this:: +away your event. To decide what to do, you can call the +:meth:`~Logger.isEnabledFor` method which takes a level argument and returns +true if the event would be created by the Logger for that level of call. +You can write code like this:: if logger.isEnabledFor(logging.DEBUG): logger.debug('Message with %s, %s', expensive_func1(), diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst index c4b3f71af75..e0083edc51c 100644 --- a/Doc/howto/sockets.rst +++ b/Doc/howto/sockets.rst @@ -19,12 +19,6 @@ Sockets ======= -Sockets are used nearly everywhere, but are one of the most severely -misunderstood technologies around. This is a 10,000 foot overview of sockets. -It's not really a tutorial - you'll still have work to do in getting things -working. It doesn't cover the fine points (and there are a lot of them), but I -hope it will give you enough background to begin using them decently. - I'm only going to talk about INET sockets, but they account for at least 99% of the sockets in use. And I'll only talk about STREAM sockets - unless you really know what you're doing (in which case this HOWTO isn't for you!), you'll get @@ -88,9 +82,11 @@ creates a "server socket":: serversocket.listen(5) A couple things to notice: we used ``socket.gethostname()`` so that the socket -would be visible to the outside world. If we had used ``s.bind(('', 80))`` or -``s.bind(('localhost', 80))`` or ``s.bind(('127.0.0.1', 80))`` we would still -have a "server" socket, but one that was only visible within the same machine. +would be visible to the outside world. If we had used ``s.bind(('localhost', +80))`` or ``s.bind(('127.0.0.1', 80))`` we would still have a "server" socket, +but one that was only visible within the same machine. ``s.bind(('', 80))`` +specifies that the socket is reachable by any address the machine happens to +have. A second thing to note: low number ports are usually reserved for "well known" services (HTTP, SNMP etc). If you're playing around, use a nice high number (4 diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index a8553081b57..f84bf0810f6 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -489,7 +489,8 @@ than the URL you pass to .add_password() will also match. :: In the above example we only supplied our ``HTTPBasicAuthHandler`` to ``build_opener``. By default openers have the handlers for normal situations - -- ``ProxyHandler``, ``UnknownHandler``, ``HTTPHandler``, + -- ``ProxyHandler`` (if a proxy setting such as an :envvar:`http_proxy` + environment variable is set), ``UnknownHandler``, ``HTTPHandler``, ``HTTPDefaultErrorHandler``, ``HTTPRedirectHandler``, ``FTPHandler``, ``FileHandler``, ``HTTPErrorProcessor``. @@ -506,10 +507,11 @@ Proxies ======= **urllib2** will auto-detect your proxy settings and use those. This is through -the ``ProxyHandler`` which is part of the normal handler chain. Normally that's -a good thing, but there are occasions when it may not be helpful [#]_. One way -to do this is to setup our own ``ProxyHandler``, with no proxies defined. This -is done using similar steps to setting up a `Basic Authentication`_ handler : :: +the ``ProxyHandler``, which is part of the normal handler chain when a proxy +setting is detected. Normally that's a good thing, but there are occasions +when it may not be helpful [#]_. One way to do this is to setup our own +``ProxyHandler``, with no proxies defined. This is done using similar steps to +setting up a `Basic Authentication`_ handler : :: >>> proxy_support = urllib2.ProxyHandler({}) >>> opener = urllib2.build_opener(proxy_support) diff --git a/Doc/install/index.rst b/Doc/install/index.rst index 8b51b97e11a..159f84816da 100644 --- a/Doc/install/index.rst +++ b/Doc/install/index.rst @@ -1042,7 +1042,7 @@ These compilers require some special libraries. This task is more complex than for Borland's C++, because there is no program to convert the library. First you have to create a list of symbols which the Python DLL exports. (You can find a good program for this task at -http://www.emmestech.com/software/pexports-0.43/download_pexports.html). +http://sourceforge.net/projects/mingw/files/MinGW/Extension/pexports/). .. I don't understand what the next line means. --amk .. (inclusive the references on data structures.) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index d18b51b6d55..489df839789 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -46,7 +46,7 @@ produces either the sum or the max:: Assuming the Python code above is saved into a file called ``prog.py``, it can be run at the command line and provides useful help messages:: - $ prog.py -h + $ python prog.py -h usage: prog.py [-h] [--sum] N [N ...] Process some integers. @@ -61,15 +61,15 @@ be run at the command line and provides useful help messages:: When run with the appropriate arguments, it prints either the sum or the max of the command-line integers:: - $ prog.py 1 2 3 4 + $ python prog.py 1 2 3 4 4 - $ prog.py 1 2 3 4 --sum + $ python prog.py 1 2 3 4 --sum 10 If invalid arguments are passed in, it will issue an error:: - $ prog.py a b c + $ python prog.py a b c usage: prog.py [-h] [--sum] N [N ...] prog.py: error: argument N: invalid int value: 'a' @@ -137,40 +137,136 @@ ArgumentParser objects argument_default=None, conflict_handler='error', \ add_help=True) - Create a new :class:`ArgumentParser` object. Each parameter has its own more - detailed description below, but in short they are: + Create a new :class:`ArgumentParser` object. All parameters should be passed + as keyword arguments. Each parameter has its own more detailed description + below, but in short they are: - * description_ - Text to display before the argument help. + * prog_ - The name of the program (default: ``sys.argv[0]``) - * epilog_ - Text to display after the argument help. + * usage_ - The string describing the program usage (default: generated from + arguments added to parser) - * add_help_ - Add a -h/--help option to the parser. (default: ``True``) + * description_ - Text to display before the argument help (default: none) - * argument_default_ - Set the global default value for arguments. - (default: ``None``) + * epilog_ - Text to display after the argument help (default: none) * parents_ - A list of :class:`ArgumentParser` objects whose arguments should - also be included. + also be included + + * formatter_class_ - A class for customizing the help output - * prefix_chars_ - The set of characters that prefix optional arguments. + * prefix_chars_ - The set of characters that prefix optional arguments (default: '-') * fromfile_prefix_chars_ - The set of characters that prefix files from - which additional arguments should be read. (default: ``None``) - - * formatter_class_ - A class for customizing the help output. + which additional arguments should be read (default: ``None``) - * conflict_handler_ - Usually unnecessary, defines strategy for resolving - conflicting optionals. + * argument_default_ - The global default value for arguments + (default: ``None``) - * prog_ - The name of the program (default: - ``sys.argv[0]``) + * conflict_handler_ - The strategy for resolving conflicting optionals + (usually unnecessary) - * usage_ - The string describing the program usage (default: generated) + * add_help_ - Add a -h/--help option to the parser (default: ``True``) The following sections describe how each of these are used. +prog +^^^^ + +By default, :class:`ArgumentParser` objects uses ``sys.argv[0]`` to determine +how to display the name of the program in help messages. This default is almost +always desirable because it will make the help messages match how the program was +invoked on the command line. For example, consider a file named +``myprogram.py`` with the following code:: + + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('--foo', help='foo help') + args = parser.parse_args() + +The help for this program will display ``myprogram.py`` as the program name +(regardless of where the program was invoked from):: + + $ python myprogram.py --help + usage: myprogram.py [-h] [--foo FOO] + + optional arguments: + -h, --help show this help message and exit + --foo FOO foo help + $ cd .. + $ python subdir\myprogram.py --help + usage: myprogram.py [-h] [--foo FOO] + + optional arguments: + -h, --help show this help message and exit + --foo FOO foo help + +To change this default behavior, another value can be supplied using the +``prog=`` argument to :class:`ArgumentParser`:: + + >>> parser = argparse.ArgumentParser(prog='myprogram') + >>> parser.print_help() + usage: myprogram [-h] + + optional arguments: + -h, --help show this help message and exit + +Note that the program name, whether determined from ``sys.argv[0]`` or from the +``prog=`` argument, is available to help messages using the ``%(prog)s`` format +specifier. + +:: + + >>> parser = argparse.ArgumentParser(prog='myprogram') + >>> parser.add_argument('--foo', help='foo of the %(prog)s program') + >>> parser.print_help() + usage: myprogram [-h] [--foo FOO] + + optional arguments: + -h, --help show this help message and exit + --foo FOO foo of the myprogram program + + +usage +^^^^^ + +By default, :class:`ArgumentParser` calculates the usage message from the +arguments it contains:: + + >>> parser = argparse.ArgumentParser(prog='PROG') + >>> parser.add_argument('--foo', nargs='?', help='foo help') + >>> parser.add_argument('bar', nargs='+', help='bar help') + >>> parser.print_help() + usage: PROG [-h] [--foo [FOO]] bar [bar ...] + + positional arguments: + bar bar help + + optional arguments: + -h, --help show this help message and exit + --foo [FOO] foo help + +The default message can be overridden with the ``usage=`` keyword argument:: + + >>> parser = argparse.ArgumentParser(prog='PROG', usage='%(prog)s [options]') + >>> parser.add_argument('--foo', nargs='?', help='foo help') + >>> parser.add_argument('bar', nargs='+', help='bar help') + >>> parser.print_help() + usage: PROG [options] + + positional arguments: + bar bar help + + optional arguments: + -h, --help show this help message and exit + --foo [FOO] foo help + +The ``%(prog)s`` format specifier is available to fill in the program name in +your usage messages. + + description ^^^^^^^^^^^ @@ -218,122 +314,6 @@ line-wrapped, but this behavior can be adjusted with the formatter_class_ argument to :class:`ArgumentParser`. -add_help -^^^^^^^^ - -By default, ArgumentParser objects add an option which simply displays -the parser's help message. For example, consider a file named -``myprogram.py`` containing the following code:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('--foo', help='foo help') - args = parser.parse_args() - -If ``-h`` or ``--help`` is supplied at the command line, the ArgumentParser -help will be printed:: - - $ python myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - optional arguments: - -h, --help show this help message and exit - --foo FOO foo help - -Occasionally, it may be useful to disable the addition of this help option. -This can be achieved by passing ``False`` as the ``add_help=`` argument to -:class:`ArgumentParser`:: - - >>> parser = argparse.ArgumentParser(prog='PROG', add_help=False) - >>> parser.add_argument('--foo', help='foo help') - >>> parser.print_help() - usage: PROG [--foo FOO] - - optional arguments: - --foo FOO foo help - -The help option is typically ``-h/--help``. The exception to this is -if the ``prefix_chars=`` is specified and does not include ``-``, in -which case ``-h`` and ``--help`` are not valid options. In -this case, the first character in ``prefix_chars`` is used to prefix -the help options:: - - >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='+/') - >>> parser.print_help() - usage: PROG [+h] - - optional arguments: - +h, ++help show this help message and exit - - -prefix_chars -^^^^^^^^^^^^ - -Most command-line options will use ``-`` as the prefix, e.g. ``-f/--foo``. -Parsers that need to support different or additional prefix -characters, e.g. for options -like ``+f`` or ``/foo``, may specify them using the ``prefix_chars=`` argument -to the ArgumentParser constructor:: - - >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='-+') - >>> parser.add_argument('+f') - >>> parser.add_argument('++bar') - >>> parser.parse_args('+f X ++bar Y'.split()) - Namespace(bar='Y', f='X') - -The ``prefix_chars=`` argument defaults to ``'-'``. Supplying a set of -characters that does not include ``-`` will cause ``-f/--foo`` options to be -disallowed. - - -fromfile_prefix_chars -^^^^^^^^^^^^^^^^^^^^^ - -Sometimes, for example when dealing with a particularly long argument lists, it -may make sense to keep the list of arguments in a file rather than typing it out -at the command line. If the ``fromfile_prefix_chars=`` argument is given to the -:class:`ArgumentParser` constructor, then arguments that start with any of the -specified characters will be treated as files, and will be replaced by the -arguments they contain. For example:: - - >>> with open('args.txt', 'w') as fp: - ... fp.write('-f\nbar') - >>> parser = argparse.ArgumentParser(fromfile_prefix_chars='@') - >>> parser.add_argument('-f') - >>> parser.parse_args(['-f', 'foo', '@args.txt']) - Namespace(f='bar') - -Arguments read from a file must by default be one per line (but see also -:meth:`~ArgumentParser.convert_arg_line_to_args`) and are treated as if they -were in the same place as the original file referencing argument on the command -line. So in the example above, the expression ``['-f', 'foo', '@args.txt']`` -is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. - -The ``fromfile_prefix_chars=`` argument defaults to ``None``, meaning that -arguments will never be treated as file references. - - -argument_default -^^^^^^^^^^^^^^^^ - -Generally, argument defaults are specified either by passing a default to -:meth:`~ArgumentParser.add_argument` or by calling the -:meth:`~ArgumentParser.set_defaults` methods with a specific set of name-value -pairs. Sometimes however, it may be useful to specify a single parser-wide -default for arguments. This can be accomplished by passing the -``argument_default=`` keyword argument to :class:`ArgumentParser`. For example, -to globally suppress attribute creation on :meth:`~ArgumentParser.parse_args` -calls, we supply ``argument_default=SUPPRESS``:: - - >>> parser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS) - >>> parser.add_argument('--foo') - >>> parser.add_argument('bar', nargs='?') - >>> parser.parse_args(['--foo', '1', 'BAR']) - Namespace(bar='BAR', foo='1') - >>> parser.parse_args([]) - Namespace() - - parents ^^^^^^^ @@ -452,6 +432,74 @@ will add information about the default value of each of the arguments:: --foo FOO FOO! (default: 42) +prefix_chars +^^^^^^^^^^^^ + +Most command-line options will use ``-`` as the prefix, e.g. ``-f/--foo``. +Parsers that need to support different or additional prefix +characters, e.g. for options +like ``+f`` or ``/foo``, may specify them using the ``prefix_chars=`` argument +to the ArgumentParser constructor:: + + >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='-+') + >>> parser.add_argument('+f') + >>> parser.add_argument('++bar') + >>> parser.parse_args('+f X ++bar Y'.split()) + Namespace(bar='Y', f='X') + +The ``prefix_chars=`` argument defaults to ``'-'``. Supplying a set of +characters that does not include ``-`` will cause ``-f/--foo`` options to be +disallowed. + + +fromfile_prefix_chars +^^^^^^^^^^^^^^^^^^^^^ + +Sometimes, for example when dealing with a particularly long argument lists, it +may make sense to keep the list of arguments in a file rather than typing it out +at the command line. If the ``fromfile_prefix_chars=`` argument is given to the +:class:`ArgumentParser` constructor, then arguments that start with any of the +specified characters will be treated as files, and will be replaced by the +arguments they contain. For example:: + + >>> with open('args.txt', 'w') as fp: + ... fp.write('-f\nbar') + >>> parser = argparse.ArgumentParser(fromfile_prefix_chars='@') + >>> parser.add_argument('-f') + >>> parser.parse_args(['-f', 'foo', '@args.txt']) + Namespace(f='bar') + +Arguments read from a file must by default be one per line (but see also +:meth:`~ArgumentParser.convert_arg_line_to_args`) and are treated as if they +were in the same place as the original file referencing argument on the command +line. So in the example above, the expression ``['-f', 'foo', '@args.txt']`` +is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. + +The ``fromfile_prefix_chars=`` argument defaults to ``None``, meaning that +arguments will never be treated as file references. + + +argument_default +^^^^^^^^^^^^^^^^ + +Generally, argument defaults are specified either by passing a default to +:meth:`~ArgumentParser.add_argument` or by calling the +:meth:`~ArgumentParser.set_defaults` methods with a specific set of name-value +pairs. Sometimes however, it may be useful to specify a single parser-wide +default for arguments. This can be accomplished by passing the +``argument_default=`` keyword argument to :class:`ArgumentParser`. For example, +to globally suppress attribute creation on :meth:`~ArgumentParser.parse_args` +calls, we supply ``argument_default=SUPPRESS``:: + + >>> parser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS) + >>> parser.add_argument('--foo') + >>> parser.add_argument('bar', nargs='?') + >>> parser.parse_args(['--foo', '1', 'BAR']) + Namespace(bar='BAR', foo='1') + >>> parser.parse_args([]) + Namespace() + + conflict_handler ^^^^^^^^^^^^^^^^ @@ -489,22 +537,20 @@ action is retained as the ``-f`` action, because only the ``--foo`` option string was overridden. -prog -^^^^ +add_help +^^^^^^^^ -By default, :class:`ArgumentParser` objects uses ``sys.argv[0]`` to determine -how to display the name of the program in help messages. This default is almost -always desirable because it will make the help messages match how the program was -invoked on the command line. For example, consider a file named -``myprogram.py`` with the following code:: +By default, ArgumentParser objects add an option which simply displays +the parser's help message. For example, consider a file named +``myprogram.py`` containing the following code:: import argparse parser = argparse.ArgumentParser() parser.add_argument('--foo', help='foo help') args = parser.parse_args() -The help for this program will display ``myprogram.py`` as the program name -(regardless of where the program was invoked from):: +If ``-h`` or ``--help`` is supplied at the command line, the ArgumentParser +help will be printed:: $ python myprogram.py --help usage: myprogram.py [-h] [--foo FOO] @@ -512,76 +558,31 @@ The help for this program will display ``myprogram.py`` as the program name optional arguments: -h, --help show this help message and exit --foo FOO foo help - $ cd .. - $ python subdir\myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - optional arguments: - -h, --help show this help message and exit - --foo FOO foo help - -To change this default behavior, another value can be supplied using the -``prog=`` argument to :class:`ArgumentParser`:: - - >>> parser = argparse.ArgumentParser(prog='myprogram') - >>> parser.print_help() - usage: myprogram [-h] - - optional arguments: - -h, --help show this help message and exit - -Note that the program name, whether determined from ``sys.argv[0]`` or from the -``prog=`` argument, is available to help messages using the ``%(prog)s`` format -specifier. - -:: - - >>> parser = argparse.ArgumentParser(prog='myprogram') - >>> parser.add_argument('--foo', help='foo of the %(prog)s program') - >>> parser.print_help() - usage: myprogram [-h] [--foo FOO] - - optional arguments: - -h, --help show this help message and exit - --foo FOO foo of the myprogram program - - -usage -^^^^^ -By default, :class:`ArgumentParser` calculates the usage message from the -arguments it contains:: +Occasionally, it may be useful to disable the addition of this help option. +This can be achieved by passing ``False`` as the ``add_help=`` argument to +:class:`ArgumentParser`:: - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--foo', nargs='?', help='foo help') - >>> parser.add_argument('bar', nargs='+', help='bar help') + >>> parser = argparse.ArgumentParser(prog='PROG', add_help=False) + >>> parser.add_argument('--foo', help='foo help') >>> parser.print_help() - usage: PROG [-h] [--foo [FOO]] bar [bar ...] - - positional arguments: - bar bar help + usage: PROG [--foo FOO] optional arguments: - -h, --help show this help message and exit - --foo [FOO] foo help + --foo FOO foo help -The default message can be overridden with the ``usage=`` keyword argument:: +The help option is typically ``-h/--help``. The exception to this is +if the ``prefix_chars=`` is specified and does not include ``-``, in +which case ``-h`` and ``--help`` are not valid options. In +this case, the first character in ``prefix_chars`` is used to prefix +the help options:: - >>> parser = argparse.ArgumentParser(prog='PROG', usage='%(prog)s [options]') - >>> parser.add_argument('--foo', nargs='?', help='foo help') - >>> parser.add_argument('bar', nargs='+', help='bar help') + >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='+/') >>> parser.print_help() - usage: PROG [options] - - positional arguments: - bar bar help + usage: PROG [+h] optional arguments: - -h, --help show this help message and exit - --foo [FOO] foo help - -The ``%(prog)s`` format specifier is available to fill in the program name in -your usage messages. + +h, ++help show this help message and exit The add_argument() method @@ -1431,7 +1432,10 @@ Other utilities Sub-commands ^^^^^^^^^^^^ -.. method:: ArgumentParser.add_subparsers() +.. method:: ArgumentParser.add_subparsers([title], [description], [prog], \ + [parser_class], [action], \ + [option_string], [dest], [help], \ + [metavar]) Many programs split up their functionality into a number of sub-commands, for example, the ``svn`` program can invoke sub-commands like ``svn @@ -1445,6 +1449,30 @@ Sub-commands command name and any :class:`ArgumentParser` constructor arguments, and returns an :class:`ArgumentParser` object that can be modified as usual. + Description of parameters: + + * title - title for the sub-parser group in help output; by default + "subcommands" if description is provided, otherwise uses title for + positional arguments + + * description - description for the sub-parser group in help output, by + default None + + * prog - usage information that will be displayed with sub-command help, + by default the name of the program and any positional arguments before the + subparser argument + + * parser_class - class which will be used to create sub-parser instances, by + default the class of the current parser (e.g. ArgumentParser) + + * dest - name of the attribute under which sub-command name will be + stored; by default None and no value is stored + + * help - help for sub-parser group in help output, by default None + + * metavar - string presenting available sub-commands in help; by default it + is None and presents sub-commands in form {cmd1, cmd2, ..} + Some example usage:: >>> # create the top-level parser @@ -1665,7 +1693,7 @@ Argument groups Mutual exclusion ^^^^^^^^^^^^^^^^ -.. method:: add_mutually_exclusive_group(required=False) +.. method:: ArgumentParser.add_mutually_exclusive_group(required=False) Create a mutually exclusive group. :mod:`argparse` will make sure that only one of the arguments in the mutually exclusive group was present on the diff --git a/Doc/library/array.rst b/Doc/library/array.rst index d34cf382b1e..1766d47c0fa 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -268,9 +268,7 @@ Examples:: Packing and unpacking of External Data Representation (XDR) data as used in some remote procedure call systems. - `The Numerical Python Manual `_ + `The Numerical Python Documentation `_ The Numeric Python extension (NumPy) defines another array type; see - http://numpy.sourceforge.net/ for further information about Numerical Python. - (A PDF version of the NumPy manual is available at - http://numpy.sourceforge.net/numdoc/numdoc.pdf). + http://www.numpy.org/ for further information about Numerical Python. diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 5130d0045d5..4db40609365 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -131,10 +131,10 @@ and classes for traversing abstract syntax trees: .. function:: literal_eval(node_or_string) - Safely evaluate an expression node or a string containing a Python - expression. The string or node provided may only consist of the following - Python literal structures: strings, numbers, tuples, lists, dicts, booleans, - and ``None``. + Safely evaluate an expression node or a Unicode or *Latin-1* encoded string + containing a Python expression. The string or node provided may only consist + of the following Python literal structures: strings, numbers, tuples, lists, + dicts, booleans, and ``None``. This can be used for safely evaluating strings containing Python expressions from untrusted sources without the need to parse the values oneself. diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 37d8d500ef1..0b5e121fe63 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -15,13 +15,14 @@ The :mod:`atexit` module defines a single function to register cleanup functions. Functions thus registered are automatically executed upon normal -interpreter termination. The order in which the functions are called is not -defined; if you have cleanup operations that depend on each other, you should -wrap them in a function and register that one. This keeps :mod:`atexit` simple. - -Note: the functions registered via this module are not called when the program -is killed by a signal not handled by Python, when a Python fatal internal error -is detected, or when :func:`os._exit` is called. +interpreter termination. :mod:`atexit` runs these functions in the *reverse* +order in which they were registered; if you register ``A``, ``B``, and ``C``, +at interpreter termination time they will be run in the order ``C``, ``B``, +``A``. + +**Note:** The functions registered via this module are not called when the +program is killed by a signal not handled by Python, when a Python fatal +internal error is detected, or when :func:`os._exit` is called. .. index:: single: exitfunc (in sys) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index ce850f6c067..d283edeb4cf 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -66,7 +66,7 @@ It defines the following functions: ``factory(stream, errors='strict')`` The factory functions must return objects providing the interfaces defined by - the base classes :class:`StreamWriter` and :class:`StreamReader`, respectively. + the base classes :class:`StreamReader` and :class:`StreamWriter`, respectively. Stream codecs can maintain state. Possible values for errors are @@ -653,7 +653,7 @@ compatible with the Python codec registry. Read one line from the input stream and return the decoded data. *size*, if given, is passed as size argument to the stream's - :meth:`readline` method. + :meth:`read` method. If *keepends* is false line-endings will be stripped from the lines returned. @@ -1098,86 +1098,112 @@ particular, the following variants typically exist: | utf_8_sig | | all languages | +-----------------+--------------------------------+--------------------------------+ -A number of codecs are specific to Python, so their codec names have no meaning -outside Python. Some of them don't convert from Unicode strings to byte strings, -but instead use the property of the Python codecs machinery that any bijective -function with one argument can be considered as an encoding. - -For the codecs listed below, the result in the "encoding" direction is always a -byte string. The result of the "decoding" direction is listed as operand type in -the table. - -.. tabularcolumns:: |l|p{0.3\linewidth}|l|p{0.3\linewidth}| - -+--------------------+---------------------------+----------------+---------------------------+ -| Codec | Aliases | Operand type | Purpose | -+====================+===========================+================+===========================+ -| base64_codec | base64, base-64 | byte string | Convert operand to MIME | -| | | | base64 | -+--------------------+---------------------------+----------------+---------------------------+ -| bz2_codec | bz2 | byte string | Compress the operand | -| | | | using bz2 | -+--------------------+---------------------------+----------------+---------------------------+ -| hex_codec | hex | byte string | Convert operand to | -| | | | hexadecimal | -| | | | representation, with two | -| | | | digits per byte | -+--------------------+---------------------------+----------------+---------------------------+ -| idna | | Unicode string | Implements :rfc:`3490`, | -| | | | see also | -| | | | :mod:`encodings.idna` | -+--------------------+---------------------------+----------------+---------------------------+ -| mbcs | dbcs | Unicode string | Windows only: Encode | -| | | | operand according to the | -| | | | ANSI codepage (CP_ACP) | -+--------------------+---------------------------+----------------+---------------------------+ -| palmos | | Unicode string | Encoding of PalmOS 3.5 | -+--------------------+---------------------------+----------------+---------------------------+ -| punycode | | Unicode string | Implements :rfc:`3492` | -+--------------------+---------------------------+----------------+---------------------------+ -| quopri_codec | quopri, quoted-printable, | byte string | Convert operand to MIME | -| | quotedprintable | | quoted printable | -+--------------------+---------------------------+----------------+---------------------------+ -| raw_unicode_escape | | Unicode string | Produce a string that is | -| | | | suitable as raw Unicode | -| | | | literal in Python source | -| | | | code | -+--------------------+---------------------------+----------------+---------------------------+ -| rot_13 | rot13 | Unicode string | Returns the Caesar-cypher | -| | | | encryption of the operand | -+--------------------+---------------------------+----------------+---------------------------+ -| string_escape | | byte string | Produce a string that is | -| | | | suitable as string | -| | | | literal in Python source | -| | | | code | -+--------------------+---------------------------+----------------+---------------------------+ -| undefined | | any | Raise an exception for | -| | | | all conversions. Can be | -| | | | used as the system | -| | | | encoding if no automatic | -| | | | :term:`coercion` between | -| | | | byte and Unicode strings | -| | | | is desired. | -+--------------------+---------------------------+----------------+---------------------------+ -| unicode_escape | | Unicode string | Produce a string that is | -| | | | suitable as Unicode | -| | | | literal in Python source | -| | | | code | -+--------------------+---------------------------+----------------+---------------------------+ -| unicode_internal | | Unicode string | Return the internal | -| | | | representation of the | -| | | | operand | -+--------------------+---------------------------+----------------+---------------------------+ -| uu_codec | uu | byte string | Convert the operand using | -| | | | uuencode | -+--------------------+---------------------------+----------------+---------------------------+ -| zlib_codec | zip, zlib | byte string | Compress the operand | -| | | | using gzip | -+--------------------+---------------------------+----------------+---------------------------+ +Python Specific Encodings +------------------------- + +A number of predefined codecs are specific to Python, so their codec names have +no meaning outside Python. These are listed in the tables below based on the +expected input and output types (note that while text encodings are the most +common use case for codecs, the underlying codec infrastructure supports +arbitrary data transforms rather than just text encodings). For asymmetric +codecs, the stated purpose describes the encoding direction. + +The following codecs provide unicode-to-str encoding [#encoding-note]_ and +str-to-unicode decoding [#decoding-note]_, similar to the Unicode text +encodings. + +.. tabularcolumns:: |l|L|L| + ++--------------------+---------------------------+---------------------------+ +| Codec | Aliases | Purpose | ++====================+===========================+===========================+ +| idna | | Implements :rfc:`3490`, | +| | | see also | +| | | :mod:`encodings.idna` | ++--------------------+---------------------------+---------------------------+ +| mbcs | dbcs | Windows only: Encode | +| | | operand according to the | +| | | ANSI codepage (CP_ACP) | ++--------------------+---------------------------+---------------------------+ +| palmos | | Encoding of PalmOS 3.5 | ++--------------------+---------------------------+---------------------------+ +| punycode | | Implements :rfc:`3492` | ++--------------------+---------------------------+---------------------------+ +| raw_unicode_escape | | Produce a string that is | +| | | suitable as raw Unicode | +| | | literal in Python source | +| | | code | ++--------------------+---------------------------+---------------------------+ +| rot_13 | rot13 | Returns the Caesar-cypher | +| | | encryption of the operand | ++--------------------+---------------------------+---------------------------+ +| undefined | | Raise an exception for | +| | | all conversions. Can be | +| | | used as the system | +| | | encoding if no automatic | +| | | :term:`coercion` between | +| | | byte and Unicode strings | +| | | is desired. | ++--------------------+---------------------------+---------------------------+ +| unicode_escape | | Produce a string that is | +| | | suitable as Unicode | +| | | literal in Python source | +| | | code | ++--------------------+---------------------------+---------------------------+ +| unicode_internal | | Return the internal | +| | | representation of the | +| | | operand | ++--------------------+---------------------------+---------------------------+ .. versionadded:: 2.3 The ``idna`` and ``punycode`` encodings. +The following codecs provide str-to-str encoding and decoding +[#decoding-note]_. + +.. tabularcolumns:: |l|L|L|L| + ++--------------------+---------------------------+---------------------------+------------------------------+ +| Codec | Aliases | Purpose | Encoder/decoder | ++====================+===========================+===========================+==============================+ +| base64_codec | base64, base-64 | Convert operand to MIME | :meth:`base64.b64encode`, | +| | | base64 (the result always | :meth:`base64.b64decode` | +| | | includes a trailing | | +| | | ``'\n'``) | | ++--------------------+---------------------------+---------------------------+------------------------------+ +| bz2_codec | bz2 | Compress the operand | :meth:`bz2.compress`, | +| | | using bz2 | :meth:`bz2.decompress` | ++--------------------+---------------------------+---------------------------+------------------------------+ +| hex_codec | hex | Convert operand to | :meth:`base64.b16encode`, | +| | | hexadecimal | :meth:`base64.b16decode` | +| | | representation, with two | | +| | | digits per byte | | ++--------------------+---------------------------+---------------------------+------------------------------+ +| quopri_codec | quopri, quoted-printable, | Convert operand to MIME | :meth:`quopri.encodestring`, | +| | quotedprintable | quoted printable | :meth:`quopri.decodestring` | ++--------------------+---------------------------+---------------------------+------------------------------+ +| string_escape | | Produce a string that is | | +| | | suitable as string | | +| | | literal in Python source | | +| | | code | | ++--------------------+---------------------------+---------------------------+------------------------------+ +| uu_codec | uu | Convert the operand using | :meth:`uu.encode`, | +| | | uuencode | :meth:`uu.decode` | ++--------------------+---------------------------+---------------------------+------------------------------+ +| zlib_codec | zip, zlib | Compress the operand | :meth:`zlib.compress`, | +| | | using gzip | :meth:`zlib.decompress` | ++--------------------+---------------------------+---------------------------+------------------------------+ + +.. [#encoding-note] str objects are also accepted as input in place of unicode + objects. They are implicitly converted to unicode by decoding them using + the default encoding. If this conversion fails, it may lead to encoding + operations raising :exc:`UnicodeDecodeError`. + +.. [#decoding-note] unicode objects are also accepted as input in place of str + objects. They are implicitly converted to str by encoding them using the + default encoding. If this conversion fails, it may lead to decoding + operations raising :exc:`UnicodeEncodeError`. + :mod:`encodings.idna` --- Internationalized Domain Names in Applications ------------------------------------------------------------------------ diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 50115ab6857..838c507caec 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -146,7 +146,7 @@ Common patterns for working with :class:`Counter` objects:: dict(c) # convert to a regular dictionary c.items() # convert to a list of (elem, cnt) pairs Counter(dict(list_of_pairs)) # convert from a list of (elem, cnt) pairs - c.most_common()[:-n:-1] # n least common elements + c.most_common()[:-n-1:-1] # n least common elements c += Counter() # remove zero and negative counts Several mathematical operations are provided for combining :class:`Counter` @@ -628,9 +628,7 @@ Example: 'Return a new OrderedDict which maps field names to their values' return OrderedDict(zip(self._fields, self)) - __dict__ = property(_asdict) - - def _replace(_self, **kwds): + def _replace(_self, **kwds): 'Return a new Point object replacing specified fields with new values' result = _self._make(map(kwds.pop, ('x', 'y'), _self)) if kwds: @@ -640,6 +638,12 @@ Example: def __getnewargs__(self): 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) + + __dict__ = _property(_asdict) + + def __getstate__(self): + 'Exclude the OrderedDict from pickling' + pass x = _property(_itemgetter(0), doc='Alias for field number 0') diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index cf0d5f85a0a..0e79a5ab5d6 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -127,7 +127,7 @@ subdirectory and all its subdirectories:: # Perform same compilation, excluding files in .svn directories. import re - compileall.compile_dir('Lib/', rx=re.compile('/[.]svn'), force=True) + compileall.compile_dir('Lib/', rx=re.compile(r'[/\\][.]svn'), force=True) .. seealso:: diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index c72a0b5a4ed..6be8debaf02 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1333,7 +1333,7 @@ returns the full pathname, but since there is no predefined naming scheme a call like ``find_library("c")`` will fail and return ``None``. If wrapping a shared library with :mod:`ctypes`, it *may* be better to determine -the shared library name at development type, and hardcode that into the wrapper +the shared library name at development time, and hardcode that into the wrapper module instead of using :func:`find_library` to locate the library at runtime. diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 409fd8bead2..4e922f03a08 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -551,8 +551,16 @@ Instance methods: .. method:: date.strftime(format) Return a string representing the date, controlled by an explicit format string. - Format codes referring to hours, minutes or seconds will see 0 values. See - section :ref:`strftime-strptime-behavior`. + Format codes referring to hours, minutes or seconds will see 0 values. For a + complete list of formatting directives, see section + :ref:`strftime-strptime-behavior`. + + +.. method:: date.__format__(format) + + Same as :meth:`.date.strftime`. This makes it possible to specify format + string for a :class:`.date` object when using :meth:`str.format`. + See section :ref:`strftime-strptime-behavior`. Example of counting days to an event:: @@ -605,6 +613,8 @@ Example of working with :class:`date`: '11/03/02' >>> d.strftime("%A %d. %B %Y") 'Monday 11. March 2002' + >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month") + 'The day is 11, the month is March.' .. _datetime-datetime: @@ -721,7 +731,8 @@ Other constructors, all class methods: *format*. This is equivalent to ``datetime(*(time.strptime(date_string, format)[0:6]))``. :exc:`ValueError` is raised if the date_string and format can't be parsed by :func:`time.strptime` or if it returns a value which isn't a - time tuple. See section :ref:`strftime-strptime-behavior`. + time tuple. For a complete list of formatting directives, see section + :ref:`strftime-strptime-behavior`. .. versionadded:: 2.5 @@ -1041,7 +1052,15 @@ Instance methods: .. method:: datetime.strftime(format) Return a string representing the date and time, controlled by an explicit format - string. See section :ref:`strftime-strptime-behavior`. + string. For a complete list of formatting directives, see section + :ref:`strftime-strptime-behavior`. + + +.. method:: datetime.__format__(format) + + Same as :meth:`.datetime.strftime`. This makes it possible to specify format + string for a :class:`.datetime` object when using :meth:`str.format`. + See section :ref:`strftime-strptime-behavior`. Examples of working with datetime objects: @@ -1088,6 +1107,8 @@ Examples of working with datetime objects: >>> # Formatting datetime >>> dt.strftime("%A, %d. %B %Y %I:%M%p") 'Tuesday, 21. November 2006 04:30PM' + >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(dt, "day", "month", "time") + 'The day is 21, the month is November, the time is 04:30PM.' Using datetime with tzinfo: @@ -1265,6 +1286,14 @@ Instance methods: .. method:: time.strftime(format) Return a string representing the time, controlled by an explicit format string. + For a complete list of formatting directives, see section + :ref:`strftime-strptime-behavior`. + + +.. method:: time.__format__(format) + + Same as :meth:`.time.strftime`. This makes it possible to specify format string + for a :class:`.time` object when using :meth:`str.format`. See section :ref:`strftime-strptime-behavior`. @@ -1314,6 +1343,8 @@ Example: 'Europe/Prague' >>> t.strftime("%H:%M:%S %Z") '12:10:30 Europe/Prague' + >>> 'The {} is {:%H:%M}.'.format("time", t) + 'The time is 12:10.' .. _datetime-tzinfo: @@ -1570,27 +1601,6 @@ For :class:`date` objects, the format codes for hours, minutes, seconds, and microseconds should not be used, as :class:`date` objects have no such values. If they're used anyway, ``0`` is substituted for them. -.. versionadded:: 2.6 - :class:`.time` and :class:`.datetime` objects support a ``%f`` format code - which expands to the number of microseconds in the object, zero-padded on - the left to six places. - -For a naive object, the ``%z`` and ``%Z`` format codes are replaced by empty -strings. - -For an aware object: - -``%z`` - :meth:`utcoffset` is transformed into a 5-character string of the form +HHMM or - -HHMM, where HH is a 2-digit string giving the number of UTC offset hours, and - MM is a 2-digit string giving the number of UTC offset minutes. For example, if - :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is - replaced with the string ``'-0330'``. - -``%Z`` - If :meth:`tzname` returns ``None``, ``%Z`` is replaced by an empty string. - Otherwise ``%Z`` is replaced by the returned value, which must be a string. - The full set of format codes supported varies across platforms, because Python calls the platform C library's :func:`strftime` function, and platform variations are common. @@ -1603,119 +1613,154 @@ format codes. The exact range of years for which :meth:`strftime` works also varies across platforms. Regardless of platform, years before 1900 cannot be used. -+-----------+--------------------------------+-------+ -| Directive | Meaning | Notes | -+===========+================================+=======+ -| ``%a`` | Locale's abbreviated weekday | | -| | name. | | -+-----------+--------------------------------+-------+ -| ``%A`` | Locale's full weekday name. | | -+-----------+--------------------------------+-------+ -| ``%b`` | Locale's abbreviated month | | -| | name. | | -+-----------+--------------------------------+-------+ -| ``%B`` | Locale's full month name. | | -+-----------+--------------------------------+-------+ -| ``%c`` | Locale's appropriate date and | | -| | time representation. | | -+-----------+--------------------------------+-------+ -| ``%d`` | Day of the month as a decimal | | -| | number [01,31]. | | -+-----------+--------------------------------+-------+ -| ``%f`` | Microsecond as a decimal | \(1) | -| | number [0,999999], zero-padded | | -| | on the left | | -+-----------+--------------------------------+-------+ -| ``%H`` | Hour (24-hour clock) as a | | -| | decimal number [00,23]. | | -+-----------+--------------------------------+-------+ -| ``%I`` | Hour (12-hour clock) as a | | -| | decimal number [01,12]. | | -+-----------+--------------------------------+-------+ -| ``%j`` | Day of the year as a decimal | | -| | number [001,366]. | | -+-----------+--------------------------------+-------+ -| ``%m`` | Month as a decimal number | | -| | [01,12]. | | -+-----------+--------------------------------+-------+ -| ``%M`` | Minute as a decimal number | | -| | [00,59]. | | -+-----------+--------------------------------+-------+ -| ``%p`` | Locale's equivalent of either | \(2) | -| | AM or PM. | | -+-----------+--------------------------------+-------+ -| ``%S`` | Second as a decimal number | \(3) | -| | [00,61]. | | -+-----------+--------------------------------+-------+ -| ``%U`` | Week number of the year | \(4) | -| | (Sunday as the first day of | | -| | the week) as a decimal number | | -| | [00,53]. All days in a new | | -| | year preceding the first | | -| | Sunday are considered to be in | | -| | week 0. | | -+-----------+--------------------------------+-------+ -| ``%w`` | Weekday as a decimal number | | -| | [0(Sunday),6]. | | -+-----------+--------------------------------+-------+ -| ``%W`` | Week number of the year | \(4) | -| | (Monday as the first day of | | -| | the week) as a decimal number | | -| | [00,53]. All days in a new | | -| | year preceding the first | | -| | Monday are considered to be in | | -| | week 0. | | -+-----------+--------------------------------+-------+ -| ``%x`` | Locale's appropriate date | | -| | representation. | | -+-----------+--------------------------------+-------+ -| ``%X`` | Locale's appropriate time | | -| | representation. | | -+-----------+--------------------------------+-------+ -| ``%y`` | Year without century as a | | -| | decimal number [00,99]. | | -+-----------+--------------------------------+-------+ -| ``%Y`` | Year with century as a decimal | | -| | number. | | -+-----------+--------------------------------+-------+ -| ``%z`` | UTC offset in the form +HHMM | \(5) | -| | or -HHMM (empty string if the | | -| | the object is naive). | | -+-----------+--------------------------------+-------+ -| ``%Z`` | Time zone name (empty string | | -| | if the object is naive). | | -+-----------+--------------------------------+-------+ -| ``%%`` | A literal ``'%'`` character. | | -+-----------+--------------------------------+-------+ ++-----------+--------------------------------+------------------------+-------+ +| Directive | Meaning | Example | Notes | ++===========+================================+========================+=======+ +| ``%a`` | Weekday as locale's || Sun, Mon, ..., Sat | \(1) | +| | abbreviated name. | (en_US); | | +| | || So, Mo, ..., Sa | | +| | | (de_DE) | | ++-----------+--------------------------------+------------------------+-------+ +| ``%A`` | Weekday as locale's full name. || Sunday, Monday, ..., | \(1) | +| | | Saturday (en_US); | | +| | || Sonntag, Montag, ..., | | +| | | Samstag (de_DE) | | ++-----------+--------------------------------+------------------------+-------+ +| ``%w`` | Weekday as a decimal number, | 0, 1, ..., 6 | | +| | where 0 is Sunday and 6 is | | | +| | Saturday. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%d`` | Day of the month as a | 01, 02, ..., 31 | | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | +| | name. | (en_US); | | +| | || Jan, Feb, ..., Dez | | +| | | (de_DE) | | ++-----------+--------------------------------+------------------------+-------+ +| ``%B`` | Month as locale's full name. || January, February, | \(1) | +| | | ..., December (en_US);| | +| | || Januar, Februar, ..., | | +| | | Dezember (de_DE) | | ++-----------+--------------------------------+------------------------+-------+ +| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | | +| | decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%y`` | Year without century as a | 00, 01, ..., 99 | | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%Y`` | Year with century as a decimal | 1970, 1988, 2001, 2013 | | +| | number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | +| | AM or PM. || am, pm (de_DE) | \(2) | ++-----------+--------------------------------+------------------------+-------+ +| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | | +| | decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(3) | +| | decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(4) | +| | number, zero-padded on the | 999999 | | +| | left. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%z`` | UTC offset in the form +HHMM | (empty), +0000, -0400, | \(5) | +| | or -HHMM (empty string if the | +1030 | | +| | the object is naive). | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%Z`` | Time zone name (empty string | (empty), UTC, EST, CST | | +| | if the object is naive). | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%j`` | Day of the year as a | 001, 002, ..., 366 | | +| | zero-padded decimal number. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(6) | +| | (Sunday as the first day of | | | +| | the week) as a zero padded | | | +| | decimal number. All days in a | | | +| | new year preceding the first | | | +| | Sunday are considered to be in | | | +| | week 0. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(6) | +| | (Monday as the first day of | | | +| | the week) as a decimal number. | | | +| | All days in a new year | | | +| | preceding the first Monday | | | +| | are considered to be in | | | +| | week 0. | | | ++-----------+--------------------------------+------------------------+-------+ +| ``%c`` | Locale's appropriate date and || Tue Aug 16 21:30:00 | \(1) | +| | time representation. | 1988 (en_US); | | +| | || Di 16 Aug 21:30:00 | | +| | | 1988 (de_DE) | | ++-----------+--------------------------------+------------------------+-------+ +| ``%x`` | Locale's appropriate date || 08/16/88 (None); | \(1) | +| | representation. || 08/16/1988 (en_US); | | +| | || 16.08.1988 (de_DE) | | ++-----------+--------------------------------+------------------------+-------+ +| ``%X`` | Locale's appropriate time || 21:30:00 (en_US); | \(1) | +| | representation. || 21:30:00 (de_DE) | | ++-----------+--------------------------------+------------------------+-------+ +| ``%%`` | A literal ``'%'`` character. | % | | ++-----------+--------------------------------+------------------------+-------+ Notes: (1) - When used with the :meth:`strptime` method, the ``%f`` directive - accepts from one to six digits and zero pads on the right. ``%f`` is - an extension to the set of format characters in the C standard (but - implemented separately in datetime objects, and therefore always - available). + Because the format depends on the current locale, care should be taken when + making assumptions about the output value. Field orderings will vary (for + example, "month/day/year" versus "day/month/year"), and the output may + contain Unicode characters encoded using the locale's default encoding (for + example, if the current locale is ``js_JP``, the default encoding could be + any one of ``eucJP``, ``SJIS``, or ``utf-8``; use :meth:`locale.getlocale` + to determine the current locale's encoding). (2) When used with the :meth:`strptime` method, the ``%p`` directive only affects the output hour field if the ``%I`` directive is used to parse the hour. (3) - The range really is ``0`` to ``61``; according to the Posix standard this - accounts for leap seconds and the (very rare) double leap seconds. - The :mod:`time` module may produce and does accept leap seconds since - it is based on the Posix standard, but the :mod:`datetime` module - does not accept leap seconds in :meth:`strptime` input nor will it - produce them in :func:`strftime` output. + Unlike the :mod:`time` module, the :mod:`datetime` module does not support + leap seconds. (4) - When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used in - calculations when the day of the week and the year are specified. + ``%f`` is an extension to the set of format characters in the C standard + (but implemented separately in datetime objects, and therefore always + available). When used with the :meth:`strptime` method, the ``%f`` + directive accepts from one to six digits and zero pads on the right. + + .. versionadded:: 2.6 (5) - For example, if :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, - ``%z`` is replaced with the string ``'-0330'``. + For a naive object, the ``%z`` and ``%Z`` format codes are replaced by empty + strings. + + For an aware object: + + ``%z`` + :meth:`utcoffset` is transformed into a 5-character string of the form + +HHMM or -HHMM, where HH is a 2-digit string giving the number of UTC + offset hours, and MM is a 2-digit string giving the number of UTC offset + minutes. For example, if :meth:`utcoffset` returns + ``timedelta(hours=-3, minutes=-30)``, ``%z`` is replaced with the string + ``'-0330'``. + + ``%Z`` + If :meth:`tzname` returns ``None``, ``%Z`` is replaced by an empty + string. Otherwise ``%Z`` is replaced by the returned value, which must + be a string. + +(6) + When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used + in calculations when the day of the week and the year are specified. .. rubric:: Footnotes diff --git a/Doc/library/email.charset.rst b/Doc/library/email.charset.rst index 840aec6a14f..0ed6f0a8c1c 100644 --- a/Doc/library/email.charset.rst +++ b/Doc/library/email.charset.rst @@ -249,5 +249,5 @@ new entries to the global character set, alias, and codec registries: *charset* is the canonical name of a character set. *codecname* is the name of a Python codec, as appropriate for the second argument to the :func:`unicode` - built-in, or to the :meth:`encode` method of a Unicode string. + built-in, or to the :meth:`~unicode.encode` method of a Unicode string. diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 628fac92bdd..499d754fa08 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -25,7 +25,8 @@ The following exception classes are defined in the :mod:`email.errors` module: Raised under some error conditions when parsing the :rfc:`2822` headers of a message, this class is derived from :exc:`MessageParseError`. It can be raised - from the :meth:`Parser.parse` or :meth:`Parser.parsestr` methods. + from the :meth:`Parser.parse ` or + :meth:`Parser.parsestr ` methods. Situations where it can be raised include finding an envelope header after the first :rfc:`2822` header of the message, finding a continuation line before the @@ -37,7 +38,8 @@ The following exception classes are defined in the :mod:`email.errors` module: Raised under some error conditions when parsing the :rfc:`2822` headers of a message, this class is derived from :exc:`MessageParseError`. It can be raised - from the :meth:`Parser.parse` or :meth:`Parser.parsestr` methods. + from the :meth:`Parser.parse ` or + :meth:`Parser.parsestr ` methods. Situations where it can be raised include not being able to find the starting or terminating boundary in a :mimetype:`multipart/\*` message when strict parsing @@ -46,19 +48,20 @@ The following exception classes are defined in the :mod:`email.errors` module: .. exception:: MultipartConversionError() - Raised when a payload is added to a :class:`Message` object using - :meth:`add_payload`, but the payload is already a scalar and the message's - :mailheader:`Content-Type` main type is not either :mimetype:`multipart` or - missing. :exc:`MultipartConversionError` multiply inherits from - :exc:`MessageError` and the built-in :exc:`TypeError`. + Raised when a payload is added to a :class:`~email.message.Message` object + using :meth:`add_payload`, but the payload is already a scalar and the + message's :mailheader:`Content-Type` main type is not either + :mimetype:`multipart` or missing. :exc:`MultipartConversionError` multiply + inherits from :exc:`MessageError` and the built-in :exc:`TypeError`. - Since :meth:`Message.add_payload` is deprecated, this exception is rarely raised - in practice. However the exception may also be raised if the :meth:`attach` + Since :meth:`Message.add_payload` is deprecated, this exception is rarely + raised in practice. However the exception may also be raised if the + :meth:`~email.message.Message.attach` method is called on an instance of a class derived from :class:`~email.mime.nonmultipart.MIMENonMultipart` (e.g. :class:`~email.mime.image.MIMEImage`). -Here's the list of the defects that the :class:`~email.mime.parser.FeedParser` +Here's the list of the defects that the :class:`~email.parser.FeedParser` can find while parsing messages. Note that the defects are added to the message where the problem was found, so for example, if a message nested inside a :mimetype:`multipart/alternative` had a malformed header, that nested message @@ -86,7 +89,7 @@ this class is *not* an exception! or was otherwise malformed. * :class:`MultipartInvariantViolationDefect` -- A message claimed to be a - :mimetype:`multipart`, but no subparts were found. Note that when a message has - this defect, its :meth:`is_multipart` method may return false even though its - content type claims to be :mimetype:`multipart`. + :mimetype:`multipart`, but no subparts were found. Note that when a message + has this defect, its :meth:`~email.message.Message.is_multipart` method may + return false even though its content type claims to be :mimetype:`multipart`. diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst index cb9cd1eb7af..4e585fce385 100644 --- a/Doc/library/email.header.rst +++ b/Doc/library/email.header.rst @@ -103,7 +103,7 @@ Here is the :class:`Header` class description: not provoke a :exc:`UnicodeError` is used. Optional *errors* is passed through to any :func:`unicode` or - :func:`ustr.encode` call, and defaults to "strict". + :meth:`unicode.encode` call, and defaults to "strict". .. method:: encode([splitchars]) diff --git a/Doc/library/email.iterators.rst b/Doc/library/email.iterators.rst index e1a35336c94..6621d51dcf7 100644 --- a/Doc/library/email.iterators.rst +++ b/Doc/library/email.iterators.rst @@ -6,8 +6,9 @@ Iterating over a message object tree is fairly easy with the -:meth:`Message.walk` method. The :mod:`email.iterators` module provides some -useful higher level iterations over message object trees. +:meth:`Message.walk ` method. The +:mod:`email.iterators` module provides some useful higher level iterations over +message object trees. .. function:: body_line_iterator(msg[, decode]) @@ -16,9 +17,11 @@ useful higher level iterations over message object trees. string payloads line-by-line. It skips over all the subpart headers, and it skips over any subpart with a payload that isn't a Python string. This is somewhat equivalent to reading the flat text representation of the message from - a file using :meth:`readline`, skipping over all the intervening headers. + a file using :meth:`~io.TextIOBase.readline`, skipping over all the + intervening headers. - Optional *decode* is passed through to :meth:`Message.get_payload`. + Optional *decode* is passed through to :meth:`Message.get_payload + `. .. function:: typed_subpart_iterator(msg[, maintype[, subtype]]) diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index da8e41c1a31..5076af62b59 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -48,8 +48,8 @@ Here are the methods of the :class:`Message` class: Note that this method is provided as a convenience and may not always format the message the way you want. For example, by default it mangles lines that begin with ``From``. For more flexibility, instantiate a - :class:`~email.generator.Generator` instance and use its :meth:`flatten` - method directly. For example:: + :class:`~email.generator.Generator` instance and use its + :meth:`~email.generator.Generator.flatten` method directly. For example:: from cStringIO import StringIO from email.generator import Generator @@ -494,8 +494,8 @@ Here are the methods of the :class:`Message` class: Set the ``boundary`` parameter of the :mailheader:`Content-Type` header to *boundary*. :meth:`set_boundary` will always quote *boundary* if - necessary. A :exc:`HeaderParseError` is raised if the message object has - no :mailheader:`Content-Type` header. + necessary. A :exc:`~email.errors.HeaderParseError` is raised if the + message object has no :mailheader:`Content-Type` header. Note that using this method is subtly different than deleting the old :mailheader:`Content-Type` header and adding a new one with the new @@ -589,7 +589,8 @@ Here are the methods of the :class:`Message` class: .. versionchanged:: 2.5 You do not need to set the epilogue to the empty string in order for the - :class:`Generator` to print a newline at the end of the file. + :class:`~email.generator.Generator` to print a newline at the end of the + file. .. attribute:: defects diff --git a/Doc/library/email.mime.rst b/Doc/library/email.mime.rst index 957504da44d..dcf7b59c05e 100644 --- a/Doc/library/email.mime.rst +++ b/Doc/library/email.mime.rst @@ -35,7 +35,8 @@ Here are the classes: *_maintype* is the :mailheader:`Content-Type` major type (e.g. :mimetype:`text` or :mimetype:`image`), and *_subtype* is the :mailheader:`Content-Type` minor type (e.g. :mimetype:`plain` or :mimetype:`gif`). *_params* is a parameter - key/value dictionary and is passed directly to :meth:`Message.add_header`. + key/value dictionary and is passed directly to :meth:`Message.add_header + `. The :class:`MIMEBase` class always adds a :mailheader:`Content-Type` header (based on *_maintype*, *_subtype*, and *_params*), and a @@ -50,8 +51,9 @@ Here are the classes: A subclass of :class:`~email.mime.base.MIMEBase`, this is an intermediate base class for MIME messages that are not :mimetype:`multipart`. The primary - purpose of this class is to prevent the use of the :meth:`attach` method, - which only makes sense for :mimetype:`multipart` messages. If :meth:`attach` + purpose of this class is to prevent the use of the + :meth:`~email.message.Message.attach` method, which only makes sense for + :mimetype:`multipart` messages. If :meth:`~email.message.Message.attach` is called, a :exc:`~email.errors.MultipartConversionError` exception is raised. .. versionadded:: 2.2.2 @@ -76,7 +78,8 @@ Here are the classes: *_subparts* is a sequence of initial subparts for the payload. It must be possible to convert this sequence to a list. You can always attach new subparts - to the message by using the :meth:`Message.attach` method. + to the message by using the :meth:`Message.attach + ` method. Additional parameters for the :mailheader:`Content-Type` header are taken from the keyword arguments, or passed into the *_params* argument, which is a keyword @@ -99,8 +102,10 @@ Here are the classes: Optional *_encoder* is a callable (i.e. function) which will perform the actual encoding of the data for transport. This callable takes one argument, which is - the :class:`MIMEApplication` instance. It should use :meth:`get_payload` and - :meth:`set_payload` to change the payload to encoded form. It should also add + the :class:`MIMEApplication` instance. It should use + :meth:`~email.message.Message.get_payload` and + :meth:`~email.message.Message.set_payload` to change the payload to encoded + form. It should also add any :mailheader:`Content-Transfer-Encoding` or other headers to the message object as necessary. The default encoding is base64. See the :mod:`email.encoders` module for a list of the built-in encoders. @@ -127,8 +132,10 @@ Here are the classes: Optional *_encoder* is a callable (i.e. function) which will perform the actual encoding of the audio data for transport. This callable takes one argument, - which is the :class:`MIMEAudio` instance. It should use :meth:`get_payload` and - :meth:`set_payload` to change the payload to encoded form. It should also add + which is the :class:`MIMEAudio` instance. It should use + :meth:`~email.message.Message.get_payload` and + :meth:`~email.message.Message.set_payload` to change the payload to encoded + form. It should also add any :mailheader:`Content-Transfer-Encoding` or other headers to the message object as necessary. The default encoding is base64. See the :mod:`email.encoders` module for a list of the built-in encoders. @@ -153,8 +160,10 @@ Here are the classes: Optional *_encoder* is a callable (i.e. function) which will perform the actual encoding of the image data for transport. This callable takes one argument, - which is the :class:`MIMEImage` instance. It should use :meth:`get_payload` and - :meth:`set_payload` to change the payload to encoded form. It should also add + which is the :class:`MIMEImage` instance. It should use + :meth:`~email.message.Message.get_payload` and + :meth:`~email.message.Message.set_payload` to change the payload to encoded + form. It should also add any :mailheader:`Content-Transfer-Encoding` or other headers to the message object as necessary. The default encoding is base64. See the :mod:`email.encoders` module for a list of the built-in encoders. diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index a91a7704243..0f99a2fb040 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -7,7 +7,8 @@ Message object structures can be created in one of two ways: they can be created from whole cloth by instantiating :class:`~email.message.Message` objects and -stringing them together via :meth:`attach` and :meth:`set_payload` calls, or they +stringing them together via :meth:`~email.message.Message.attach` and +:meth:`~email.message.Message.set_payload` calls, or they can be created by parsing a flat text representation of the email message. The :mod:`email` package provides a standard parser that understands most email @@ -16,8 +17,9 @@ or a file object, and the parser will return to you the root :class:`~email.message.Message` instance of the object structure. For simple, non-MIME messages the payload of this root object will likely be a string containing the text of the message. For MIME messages, the root object will -return ``True`` from its :meth:`is_multipart` method, and the subparts can be -accessed via the :meth:`get_payload` and :meth:`walk` methods. +return ``True`` from its :meth:`~email.message.Message.is_multipart` method, and +the subparts can be accessed via the :meth:`~email.message.Message.get_payload` +and :meth:`~email.message.Message.walk` methods. There are actually two parser interfaces available for use, the classic :class:`Parser` API and the incremental :class:`FeedParser` API. The classic @@ -127,7 +129,8 @@ class. Read all the data from the file-like object *fp*, parse the resulting text, and return the root message object. *fp* must support both the - :meth:`readline` and the :meth:`read` methods on file-like objects. + :meth:`~io.TextIOBase.readline` and the :meth:`~io.TextIOBase.read` + methods on file-like objects. The text contained in *fp* must be formatted as a block of :rfc:`2822` style headers and header continuation lines, optionally preceded by a @@ -147,7 +150,7 @@ class. Similar to the :meth:`parse` method, except it takes a string object instead of a file-like object. Calling this method on a string is exactly - equivalent to wrapping *text* in a :class:`StringIO` instance first and + equivalent to wrapping *text* in a :class:`~StringIO.StringIO` instance first and calling :meth:`parse`. Optional *headersonly* is as with the :meth:`parse` method. @@ -165,7 +168,7 @@ in the top-level :mod:`email` package namespace. Return a message object structure from a string. This is exactly equivalent to ``Parser().parsestr(s)``. Optional *_class* and *strict* are interpreted as - with the :class:`Parser` class constructor. + with the :class:`~email.parser.Parser` class constructor. .. versionchanged:: 2.2.2 The *strict* flag was added. @@ -175,7 +178,7 @@ in the top-level :mod:`email` package namespace. Return a message object structure tree from an open file object. This is exactly equivalent to ``Parser().parse(fp)``. Optional *_class* and *strict* - are interpreted as with the :class:`Parser` class constructor. + are interpreted as with the :class:`~email.parser.Parser` class constructor. .. versionchanged:: 2.2.2 The *strict* flag was added. @@ -193,32 +196,35 @@ Here are some notes on the parsing semantics: * Most non-\ :mimetype:`multipart` type messages are parsed as a single message object with a string payload. These objects will return ``False`` for - :meth:`is_multipart`. Their :meth:`get_payload` method will return a string - object. + :meth:`~email.message.Message.is_multipart`. Their + :meth:`~email.message.Message.get_payload` method will return a string object. * All :mimetype:`multipart` type messages will be parsed as a container message object with a list of sub-message objects for their payload. The outer - container message will return ``True`` for :meth:`is_multipart` and their - :meth:`get_payload` method will return the list of :class:`~email.message.Message` - subparts. + container message will return ``True`` for + :meth:`~email.message.Message.is_multipart` and their + :meth:`~email.message.Message.get_payload` method will return the list of + :class:`~email.message.Message` subparts. * Most messages with a content type of :mimetype:`message/\*` (e.g. :mimetype:`message/delivery-status` and :mimetype:`message/rfc822`) will also be parsed as container object containing a list payload of length 1. Their - :meth:`is_multipart` method will return ``True``. The single element in the - list payload will be a sub-message object. + :meth:`~email.message.Message.is_multipart` method will return ``True``. + The single element in the list payload will be a sub-message object. * Some non-standards compliant messages may not be internally consistent about their :mimetype:`multipart`\ -edness. Such messages may have a :mailheader:`Content-Type` header of type :mimetype:`multipart`, but their - :meth:`is_multipart` method may return ``False``. If such messages were parsed - with the :class:`FeedParser`, they will have an instance of the - :class:`MultipartInvariantViolationDefect` class in their *defects* attribute - list. See :mod:`email.errors` for details. + :meth:`~email.message.Message.is_multipart` method may return ``False``. + If such messages were parsed with the :class:`~email.parser.FeedParser`, + they will have an instance of the + :class:`~email.errors.MultipartInvariantViolationDefect` class in their + *defects* attribute list. See :mod:`email.errors` for details. .. rubric:: Footnotes .. [#] As of email package version 3.0, introduced in Python 2.4, the classic - :class:`Parser` was re-implemented in terms of the :class:`FeedParser`, so the - semantics and results are identical between the two parsers. + :class:`~email.parser.Parser` was re-implemented in terms of the + :class:`~email.parser.FeedParser`, so the semantics and results are + identical between the two parsers. diff --git a/Doc/library/email.rst b/Doc/library/email.rst index aaff1534ca4..4e90ce474f8 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -112,14 +112,15 @@ Here are the major differences between :mod:`email` version 4 and version 3: *Note that the version 3 names will continue to work until Python 2.6*. * The :mod:`email.mime.application` module was added, which contains the - :class:`MIMEApplication` class. + :class:`~email.mime.application.MIMEApplication` class. * Methods that were deprecated in version 3 have been removed. These include :meth:`Generator.__call__`, :meth:`Message.get_type`, :meth:`Message.get_main_type`, :meth:`Message.get_subtype`. * Fixes have been added for :rfc:`2231` support which can change some of the - return types for :func:`Message.get_param` and friends. Under some + return types for :func:`Message.get_param ` + and friends. Under some circumstances, values which used to return a 3-tuple now return simple strings (specifically, if all extended parameter segments were unencoded, there is no language and charset designation expected, so the return type is now a simple @@ -128,23 +129,24 @@ Here are the major differences between :mod:`email` version 4 and version 3: Here are the major differences between :mod:`email` version 3 and version 2: -* The :class:`FeedParser` class was introduced, and the :class:`Parser` class - was implemented in terms of the :class:`FeedParser`. All parsing therefore is +* The :class:`~email.parser.FeedParser` class was introduced, and the + :class:`~email.parser.Parser` class was implemented in terms of the + :class:`~email.parser.FeedParser`. All parsing therefore is non-strict, and parsing will make a best effort never to raise an exception. Problems found while parsing messages are stored in the message's *defect* attribute. * All aspects of the API which raised :exc:`DeprecationWarning`\ s in version 2 have been removed. These include the *_encoder* argument to the - :class:`MIMEText` constructor, the :meth:`Message.add_payload` method, the - :func:`Utils.dump_address_pair` function, and the functions :func:`Utils.decode` - and :func:`Utils.encode`. + :class:`~email.mime.text.MIMEText` constructor, the + :meth:`Message.add_payload` method, the :func:`Utils.dump_address_pair` + function, and the functions :func:`Utils.decode` and :func:`Utils.encode`. * New :exc:`DeprecationWarning`\ s have been added to: :meth:`Generator.__call__`, :meth:`Message.get_type`, :meth:`Message.get_main_type`, :meth:`Message.get_subtype`, and the *strict* - argument to the :class:`Parser` class. These are expected to be removed in - future versions. + argument to the :class:`~email.parser.Parser` class. These are expected to + be removed in future versions. * Support for Pythons earlier than 2.3 has been removed. @@ -152,53 +154,61 @@ Here are the differences between :mod:`email` version 2 and version 1: * The :mod:`email.Header` and :mod:`email.Charset` modules have been added. -* The pickle format for :class:`Message` instances has changed. Since this was - never (and still isn't) formally defined, this isn't considered a backward - incompatibility. However if your application pickles and unpickles - :class:`Message` instances, be aware that in :mod:`email` version 2, - :class:`Message` instances now have private variables *_charset* and - *_default_type*. +* The pickle format for :class:`~email.message.Message` instances has changed. + Since this was never (and still isn't) formally defined, this isn't + considered a backward incompatibility. However if your application pickles + and unpickles :class:`~email.message.Message` instances, be aware that in + :mod:`email` version 2, :class:`~email.message.Message` instances now have + private variables *_charset* and *_default_type*. -* Several methods in the :class:`Message` class have been deprecated, or their - signatures changed. Also, many new methods have been added. See the - documentation for the :class:`Message` class for details. The changes should be - completely backward compatible. +* Several methods in the :class:`~email.message.Message` class have been + deprecated, or their signatures changed. Also, many new methods have been + added. See the documentation for the :class:`~email.message.Message` class + for details. The changes should be completely backward compatible. * The object structure has changed in the face of :mimetype:`message/rfc822` - content types. In :mod:`email` version 1, such a type would be represented by a - scalar payload, i.e. the container message's :meth:`is_multipart` returned - false, :meth:`get_payload` was not a list object, but a single :class:`Message` - instance. + content types. In :mod:`email` version 1, such a type would be represented + by a scalar payload, i.e. the container message's + :meth:`~email.message.Message.is_multipart` returned false, + :meth:`~email.message.Message.get_payload` was not a list object, but a + single :class:`~email.message.Message` instance. This structure was inconsistent with the rest of the package, so the object representation for :mimetype:`message/rfc822` content types was changed. In :mod:`email` version 2, the container *does* return ``True`` from - :meth:`is_multipart`, and :meth:`get_payload` returns a list containing a single - :class:`Message` item. - - Note that this is one place that backward compatibility could not be completely - maintained. However, if you're already testing the return type of - :meth:`get_payload`, you should be fine. You just need to make sure your code - doesn't do a :meth:`set_payload` with a :class:`Message` instance on a container - with a content type of :mimetype:`message/rfc822`. - -* The :class:`Parser` constructor's *strict* argument was added, and its - :meth:`parse` and :meth:`parsestr` methods grew a *headersonly* argument. The - *strict* flag was also added to functions :func:`email.message_from_file` and - :func:`email.message_from_string`. - -* :meth:`Generator.__call__` is deprecated; use :meth:`Generator.flatten` - instead. The :class:`Generator` class has also grown the :meth:`clone` method. - -* The :class:`DecodedGenerator` class in the :mod:`email.Generator` module was - added. - -* The intermediate base classes :class:`MIMENonMultipart` and - :class:`MIMEMultipart` have been added, and interposed in the class hierarchy - for most of the other MIME-related derived classes. - -* The *_encoder* argument to the :class:`MIMEText` constructor has been - deprecated. Encoding now happens implicitly based on the *_charset* argument. + :meth:`~email.message.Message.is_multipart`, and + :meth:`~email.message.Message.get_payload` returns a list containing a single + :class:`~email.message.Message` item. + + Note that this is one place that backward compatibility could not be + completely maintained. However, if you're already testing the return type of + :meth:`~email.message.Message.get_payload`, you should be fine. You just need + to make sure your code doesn't do a :meth:`~email.message.Message.set_payload` + with a :class:`~email.message.Message` instance on a container with a content + type of :mimetype:`message/rfc822`. + +* The :class:`~email.parser.Parser` constructor's *strict* argument was added, + and its :meth:`~email.parser.Parser.parse` and + :meth:`~email.parser.Parser.parsestr` methods grew a *headersonly* argument. + The *strict* flag was also added to functions :func:`email.message_from_file` + and :func:`email.message_from_string`. + +* :meth:`Generator.__call__` is deprecated; use :meth:`Generator.flatten + ` instead. The + :class:`~email.generator.Generator` class has also grown the + :meth:`~email.generator.Generator.clone` method. + +* The :class:`~email.generator.DecodedGenerator` class in the + :mod:`email.generator` module was added. + +* The intermediate base classes + :class:`~email.mime.nonmultipart.MIMENonMultipart` and + :class:`~email.mime.multipart.MIMEMultipart` have been added, and interposed + in the class hierarchy for most of the other MIME-related derived classes. + +* The *_encoder* argument to the :class:`~email.mime.text.MIMEText` constructor + has been deprecated. Encoding now happens implicitly based on the + *_charset* argument. * The following functions in the :mod:`email.Utils` module have been deprecated: :func:`dump_address_pairs`, :func:`decode`, and :func:`encode`. The following @@ -231,17 +241,22 @@ package has the following differences: * :func:`messageFromFile` has been renamed to :func:`message_from_file`. -The :class:`Message` class has the following differences: +The :class:`~email.message.Message` class has the following differences: -* The method :meth:`asString` was renamed to :meth:`as_string`. +* The method :meth:`asString` was renamed to + :meth:`~email.message.Message.as_string`. -* The method :meth:`ismultipart` was renamed to :meth:`is_multipart`. +* The method :meth:`ismultipart` was renamed to + :meth:`~email.message.Message.is_multipart`. -* The :meth:`get_payload` method has grown a *decode* optional argument. +* The :meth:`~email.message.Message.get_payload` method has grown a *decode* + optional argument. -* The method :meth:`getall` was renamed to :meth:`get_all`. +* The method :meth:`getall` was renamed to + :meth:`~email.message.Message.get_all`. -* The method :meth:`addheader` was renamed to :meth:`add_header`. +* The method :meth:`addheader` was renamed to + :meth:`~email.message.Message.add_header`. * The method :meth:`gettype` was renamed to :meth:`get_type`. @@ -249,48 +264,57 @@ The :class:`Message` class has the following differences: * The method :meth:`getsubtype` was renamed to :meth:`get_subtype`. -* The method :meth:`getparams` was renamed to :meth:`get_params`. Also, whereas - :meth:`getparams` returned a list of strings, :meth:`get_params` returns a list - of 2-tuples, effectively the key/value pairs of the parameters, split on the - ``'='`` sign. +* The method :meth:`getparams` was renamed to + :meth:`~email.message.Message.get_params`. Also, whereas :meth:`getparams` + returned a list of strings, :meth:`~email.message.Message.get_params` returns + a list of 2-tuples, effectively the key/value pairs of the parameters, split + on the ``'='`` sign. -* The method :meth:`getparam` was renamed to :meth:`get_param`. +* The method :meth:`getparam` was renamed to + :meth:`~email.message.Message.get_param`. -* The method :meth:`getcharsets` was renamed to :meth:`get_charsets`. +* The method :meth:`getcharsets` was renamed to + :meth:`~email.message.Message.get_charsets`. -* The method :meth:`getfilename` was renamed to :meth:`get_filename`. +* The method :meth:`getfilename` was renamed to + :meth:`~email.message.Message.get_filename`. -* The method :meth:`getboundary` was renamed to :meth:`get_boundary`. +* The method :meth:`getboundary` was renamed to + :meth:`~email.message.Message.get_boundary`. -* The method :meth:`setboundary` was renamed to :meth:`set_boundary`. +* The method :meth:`setboundary` was renamed to + :meth:`~email.message.Message.set_boundary`. * The method :meth:`getdecodedpayload` was removed. To get similar - functionality, pass the value 1 to the *decode* flag of the get_payload() - method. + functionality, pass the value 1 to the *decode* flag of the + :meth:`~email.message.Message.get_payload` method. * The method :meth:`getpayloadastext` was removed. Similar functionality is - supported by the :class:`DecodedGenerator` class in the :mod:`email.generator` - module. + supported by the :class:`~email.generator.DecodedGenerator` class in the + :mod:`email.generator` module. * The method :meth:`getbodyastext` was removed. You can get similar - functionality by creating an iterator with :func:`typed_subpart_iterator` in the - :mod:`email.iterators` module. + functionality by creating an iterator with + :func:`~email.iterators.typed_subpart_iterator` in the :mod:`email.iterators` + module. -The :class:`Parser` class has no differences in its public interface. It does -have some additional smarts to recognize :mimetype:`message/delivery-status` -type messages, which it represents as a :class:`Message` instance containing -separate :class:`Message` subparts for each header block in the delivery status -notification [#]_. +The :class:`~email.parser.Parser` class has no differences in its public +interface. It does have some additional smarts to recognize +:mimetype:`message/delivery-status` type messages, which it represents as a +:class:`~email.message.Message` instance containing separate +:class:`~email.message.Message` subparts for each header block in the delivery +status notification [#]_. -The :class:`Generator` class has no differences in its public interface. There -is a new class in the :mod:`email.generator` module though, called -:class:`DecodedGenerator` which provides most of the functionality previously -available in the :meth:`Message.getpayloadastext` method. +The :class:`~email.generator.Generator` class has no differences in its public +interface. There is a new class in the :mod:`email.generator` module though, +called :class:`~email.generator.DecodedGenerator` which provides most of the +functionality previously available in the :meth:`Message.getpayloadastext` +method. The following modules and classes have been changed: -* The :class:`MIMEBase` class constructor arguments *_major* and *_minor* have - changed to *_maintype* and *_subtype* respectively. +* The :class:`~email.mime.base.MIMEBase` class constructor arguments *_major* + and *_minor* have changed to *_maintype* and *_subtype* respectively. * The ``Image`` class/module has been renamed to ``MIMEImage``. The *_minor* argument has been renamed to *_subtype*. @@ -303,7 +327,8 @@ The following modules and classes have been changed: but that clashed with the Python standard library module :mod:`rfc822` on some case-insensitive file systems. - Also, the :class:`MIMEMessage` class now represents any kind of MIME message + Also, the :class:`~email.mime.message.MIMEMessage` class now represents any + kind of MIME message with main type :mimetype:`message`. It takes an optional argument *_subtype* which is used to set the MIME subtype. *_subtype* defaults to :mimetype:`rfc822`. @@ -313,8 +338,8 @@ The following modules and classes have been changed: :mod:`email.utils` module. The ``MsgReader`` class/module has been removed. Its functionality is most -closely supported in the :func:`body_line_iterator` function in the -:mod:`email.iterators` module. +closely supported in the :func:`~email.iterators.body_line_iterator` function +in the :mod:`email.iterators` module. .. rubric:: Footnotes diff --git a/Doc/library/email.util.rst b/Doc/library/email.util.rst index 744f1da60bb..26f1919f527 100644 --- a/Doc/library/email.util.rst +++ b/Doc/library/email.util.rst @@ -41,8 +41,8 @@ There are several useful utilities provided in the :mod:`email.utils` module: This method returns a list of 2-tuples of the form returned by ``parseaddr()``. *fieldvalues* is a sequence of header field values as might be returned by - :meth:`Message.get_all`. Here's a simple example that gets all the recipients - of a message:: + :meth:`Message.get_all `. Here's a simple + example that gets all the recipients of a message:: from email.utils import getaddresses @@ -130,7 +130,8 @@ There are several useful utilities provided in the :mod:`email.utils` module: .. function:: collapse_rfc2231_value(value[, errors[, fallback_charset]]) When a header parameter is encoded in :rfc:`2231` format, - :meth:`Message.get_param` may return a 3-tuple containing the character set, + :meth:`Message.get_param ` may return a + 3-tuple containing the character set, language, and value. :func:`collapse_rfc2231_value` turns this into a unicode string. Optional *errors* is passed to the *errors* argument of the built-in :func:`unicode` function; it defaults to ``replace``. Optional @@ -152,11 +153,12 @@ There are several useful utilities provided in the :mod:`email.utils` module: .. versionchanged:: 2.4 The :func:`decode` function has been removed; use the - :meth:`Header.decode_header` method instead. + :meth:`Header.decode_header ` method + instead. .. versionchanged:: 2.4 - The :func:`encode` function has been removed; use the :meth:`Header.encode` - method instead. + The :func:`encode` function has been removed; use the :meth:`Header.encode + ` method instead. .. rubric:: Footnotes diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index c12e874a39f..ad75ed5fe0d 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -346,7 +346,7 @@ The following exceptions are the exceptions that are actually raised. it has another type (such as a string), the object's value is printed and the exit status is one. - Instances have an attribute :attr:`code` which is set to the proposed exit + Instances have an attribute :attr:`!code` which is set to the proposed exit status or error message (defaulting to ``None``). Also, this exception derives directly from :exc:`BaseException` and not :exc:`StandardError`, since it is not technically an error. diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index 40ae08be3e3..cf4ee9fac7e 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -1,6 +1,5 @@ - -:mod:`fcntl` --- The :func:`fcntl` and :func:`ioctl` system calls -================================================================= +:mod:`fcntl` --- The ``fcntl`` and ``ioctl`` system calls +========================================================= .. module:: fcntl :platform: Unix @@ -18,7 +17,7 @@ interface to the :c:func:`fcntl` and :c:func:`ioctl` Unix routines. All functions in this module take a file descriptor *fd* as their first argument. This can be an integer file descriptor, such as returned by ``sys.stdin.fileno()``, or a file object, such as ``sys.stdin`` itself, which -provides a :meth:`fileno` which returns a genuine file descriptor. +provides a :meth:`~io.IOBase.fileno` which returns a genuine file descriptor. The module defines the following functions: @@ -26,7 +25,8 @@ The module defines the following functions: .. function:: fcntl(fd, op[, arg]) Perform the requested operation on file descriptor *fd* (file objects providing - a :meth:`fileno` method are accepted as well). The operation is defined by *op* + a :meth:`~io.IOBase.fileno` method are accepted as well). The operation is + defined by *op* and is operating system dependent. These codes are also found in the :mod:`fcntl` module. The argument *arg* is optional, and defaults to the integer value ``0``. When present, it can either be an integer value, or a string. @@ -46,7 +46,7 @@ The module defines the following functions: .. function:: ioctl(fd, op[, arg[, mutate_flag]]) - This function is identical to the :func:`fcntl` function, except that the + This function is identical to the :func:`~fcntl.fcntl` function, except that the operations are typically defined in the library module :mod:`termios` and the argument handling is even more complicated. @@ -56,7 +56,8 @@ The module defines the following functions: integer ``0``), an object supporting the read-only buffer interface (most likely a plain Python string) or an object supporting the read-write buffer interface. - In all but the last case, behaviour is as for the :func:`fcntl` function. + In all but the last case, behaviour is as for the :func:`~fcntl.fcntl` + function. If a mutable buffer is passed, then the behaviour is determined by the value of the *mutate_flag* parameter. @@ -95,16 +96,16 @@ The module defines the following functions: .. function:: flock(fd, op) Perform the lock operation *op* on file descriptor *fd* (file objects providing - a :meth:`fileno` method are accepted as well). See the Unix manual + a :meth:`~io.IOBase.fileno` method are accepted as well). See the Unix manual :manpage:`flock(2)` for details. (On some systems, this function is emulated using :c:func:`fcntl`.) .. function:: lockf(fd, operation, [length, [start, [whence]]]) - This is essentially a wrapper around the :func:`fcntl` locking calls. *fd* is - the file descriptor of the file to lock or unlock, and *operation* is one of the - following values: + This is essentially a wrapper around the :func:`~fcntl.fcntl` locking calls. + *fd* is the file descriptor of the file to lock or unlock, and *operation* + is one of the following values: * :const:`LOCK_UN` -- unlock * :const:`LOCK_SH` -- acquire a shared lock @@ -119,13 +120,13 @@ The module defines the following functions: systems, :const:`LOCK_EX` can only be used if the file descriptor refers to a file opened for writing. - *length* is the number of bytes to lock, *start* is the byte offset at which the - lock starts, relative to *whence*, and *whence* is as with :func:`fileobj.seek`, - specifically: + *length* is the number of bytes to lock, *start* is the byte offset at + which the lock starts, relative to *whence*, and *whence* is as with + :func:`io.IOBase.seek`, specifically: - * :const:`0` -- relative to the start of the file (:const:`SEEK_SET`) - * :const:`1` -- relative to the current buffer position (:const:`SEEK_CUR`) - * :const:`2` -- relative to the end of the file (:const:`SEEK_END`) + * :const:`0` -- relative to the start of the file (:data:`os.SEEK_SET`) + * :const:`1` -- relative to the current buffer position (:data:`os.SEEK_CUR`) + * :const:`2` -- relative to the end of the file (:data:`os.SEEK_END`) The default for *start* is 0, which means to start at the beginning of the file. The default for *length* is 0 which means to lock to the end of the file. The @@ -150,7 +151,8 @@ lay-out for the *lockdata* variable is system dependent --- therefore using the .. seealso:: Module :mod:`os` - If the locking flags :const:`O_SHLOCK` and :const:`O_EXLOCK` are present - in the :mod:`os` module (on BSD only), the :func:`os.open` function - provides an alternative to the :func:`lockf` and :func:`flock` functions. + If the locking flags :data:`~os.O_SHLOCK` and :data:`~os.O_EXLOCK` are + present in the :mod:`os` module (on BSD only), the :func:`os.open` + function provides an alternative to the :func:`lockf` and :func:`flock` + functions. diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst index 172a643ac50..ce892ce277c 100644 --- a/Doc/library/fileinput.rst +++ b/Doc/library/fileinput.rst @@ -50,7 +50,7 @@ provided by this module. The following function is the primary interface of this module: -.. function:: input([files[, inplace[, backup[, mode[, openhook]]]]]) +.. function:: input([files[, inplace[, backup[, bufsize[, mode[, openhook]]]]]]) Create an instance of the :class:`FileInput` class. The instance will be used as global state for the functions of this module, and is also returned to use @@ -122,7 +122,7 @@ The class which implements the sequence behavior provided by the module is available for subclassing as well: -.. class:: FileInput([files[, inplace[, backup[, mode[, openhook]]]]]) +.. class:: FileInput([files[, inplace[, backup[,bufsize[, mode[, openhook]]]]]]) Class :class:`FileInput` is the implementation; its methods :meth:`filename`, :meth:`fileno`, :meth:`lineno`, :meth:`filelineno`, :meth:`isfirstline`, diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index 988278952b5..559e8a21ddc 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -23,16 +23,17 @@ see Internet :rfc:`959`. Here's a sample session using the :mod:`ftplib` module:: >>> from ftplib import FTP - >>> ftp = FTP('ftp.cwi.nl') # connect to host, default port - >>> ftp.login() # user anonymous, passwd anonymous@ - >>> ftp.retrlines('LIST') # list directory contents - total 24418 - drwxrwsr-x 5 ftp-usr pdmaint 1536 Mar 20 09:48 . - dr-xr-srwt 105 ftp-usr pdmaint 1536 Mar 21 14:32 .. - -rw-r--r-- 1 ftp-usr pdmaint 5305 Mar 20 09:48 INDEX - . - . - . + >>> ftp = FTP('ftp.debian.org') # connect to host, default port + >>> ftp.login() # user anonymous, passwd anonymous@ + '230 Login successful.' + >>> ftp.cwd('debian') # change into "debian" directory + >>> ftp.retrlines('LIST') # list directory contents + -rw-rw-r-- 1 1176 1176 1063 Jun 15 10:18 README + ... + drwxr-sr-x 5 1176 1176 4096 Dec 19 2000 pool + drwxr-sr-x 4 1176 1176 4096 Nov 17 2008 project + drwxr-xr-x 3 1176 1176 4096 Oct 10 2012 tools + '226 Directory send OK.' >>> ftp.retrbinary('RETR README', open('README', 'wb').write) '226 Transfer complete.' >>> ftp.quit() @@ -387,7 +388,8 @@ FTP_TLS Objects .. method:: FTP_TLS.auth() - Set up secure control connection by using TLS or SSL, depending on what specified in :meth:`ssl_version` attribute. + Set up secure control connection by using TLS or SSL, depending on what + specified in :meth:`ssl_version` attribute. .. method:: FTP_TLS.prot_p() @@ -396,5 +398,3 @@ FTP_TLS Objects .. method:: FTP_TLS.prot_c() Set up clear text data connection. - - diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 08fb7fedcc5..aabf2346bb5 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -199,8 +199,10 @@ available. They are listed here in alphabetical order. Compile the *source* into a code or AST object. Code objects can be executed by an :keyword:`exec` statement or evaluated by a call to :func:`eval`. - *source* can either be a string or an AST object. Refer to the :mod:`ast` - module documentation for information on how to work with AST objects. + *source* can either be a Unicode string, a *Latin-1* encoded string or an + AST object. + Refer to the :mod:`ast` module documentation for information on how to work + with AST objects. The *filename* argument should give the file from which the code was read; pass some recognizable value if it wasn't read from a file (``''`` is @@ -224,8 +226,8 @@ available. They are listed here in alphabetical order. Future statements are specified by bits which can be bitwise ORed together to specify multiple statements. The bitfield required to specify a given feature - can be found as the :attr:`compiler_flag` attribute on the :class:`_Feature` - instance in the :mod:`__future__` module. + can be found as the :attr:`~__future__._Feature.compiler_flag` attribute on + the :class:`~__future__._Feature` instance in the :mod:`__future__` module. This function raises :exc:`SyntaxError` if the compiled source is invalid, and :exc:`TypeError` if the source contains null bytes. @@ -388,9 +390,9 @@ available. They are listed here in alphabetical order. .. function:: eval(expression[, globals[, locals]]) - The arguments are a string and optional globals and locals. If provided, - *globals* must be a dictionary. If provided, *locals* can be any mapping - object. + The arguments are a Unicode or *Latin-1* encoded string and optional + globals and locals. If provided, *globals* must be a dictionary. + If provided, *locals* can be any mapping object. .. versionchanged:: 2.4 formerly *locals* was required to be a dictionary. @@ -699,7 +701,7 @@ available. They are listed here in alphabetical order. One useful application of the second form of :func:`iter` is to read lines of a file until a certain line is reached. The following example reads a file - until the :meth:`readline` method returns an empty string:: + until the :meth:`~io.TextIOBase.readline` method returns an empty string:: with open('mydata.txt') as fp: for line in iter(fp.readline, ''): @@ -1011,10 +1013,10 @@ available. They are listed here in alphabetical order. turns the :meth:`voltage` method into a "getter" for a read-only attribute with the same name. - A property object has :attr:`getter`, :attr:`setter`, and :attr:`deleter` - methods usable as decorators that create a copy of the property with the - corresponding accessor function set to the decorated function. This is - best explained with an example:: + A property object has :attr:`~property.getter`, :attr:`~property.setter`, + and :attr:`~property.deleter` methods usable as decorators that create a + copy of the property with the corresponding accessor function set to the + decorated function. This is best explained with an example:: class C(object): def __init__(self): @@ -1257,13 +1259,13 @@ available. They are listed here in alphabetical order. Return a :term:`slice` object representing the set of indices specified by ``range(start, stop, step)``. The *start* and *step* arguments default to - ``None``. Slice objects have read-only data attributes :attr:`start`, - :attr:`stop` and :attr:`step` which merely return the argument values (or their - default). They have no other explicit functionality; however they are used by - Numerical Python and other third party extensions. Slice objects are also - generated when extended indexing syntax is used. For example: - ``a[start:stop:step]`` or ``a[start:stop, i]``. See :func:`itertools.islice` - for an alternate version that returns an iterator. + ``None``. Slice objects have read-only data attributes :attr:`~slice.start`, + :attr:`~slice.stop` and :attr:`~slice.step` which merely return the argument + values (or their default). They have no other explicit functionality; + however they are used by Numerical Python and other third party extensions. + Slice objects are also generated when extended indexing syntax is used. For + example: ``a[start:stop:step]`` or ``a[start:stop, i]``. See + :func:`itertools.islice` for an alternate version that returns an iterator. .. function:: sorted(iterable[, cmp[, key[, reverse]]]) @@ -1368,9 +1370,10 @@ available. They are listed here in alphabetical order. been overridden in a class. The search order is same as that used by :func:`getattr` except that the *type* itself is skipped. - The :attr:`__mro__` attribute of the *type* lists the method resolution - search order used by both :func:`getattr` and :func:`super`. The attribute - is dynamic and can change whenever the inheritance hierarchy is updated. + The :attr:`~class.__mro__` attribute of the *type* lists the method + resolution search order used by both :func:`getattr` and :func:`super`. The + attribute is dynamic and can change whenever the inheritance hierarchy is + updated. If the second argument is omitted, the super object returned is unbound. If the second argument is an object, ``isinstance(obj, type)`` must be true. If @@ -1444,10 +1447,10 @@ available. They are listed here in alphabetical order. With three arguments, return a new type object. This is essentially a dynamic form of the :keyword:`class` statement. The *name* string is the - class name and becomes the :attr:`__name__` attribute; the *bases* tuple - itemizes the base classes and becomes the :attr:`__bases__` attribute; + class name and becomes the :attr:`~class.__name__` attribute; the *bases* tuple + itemizes the base classes and becomes the :attr:`~class.__bases__` attribute; and the *dict* dictionary is the namespace containing definitions for class - body and becomes the :attr:`__dict__` attribute. For example, the + body and becomes the :attr:`~object.__dict__` attribute. For example, the following two statements create identical :class:`type` objects: >>> class X(object): @@ -1511,7 +1514,7 @@ available. They are listed here in alphabetical order. .. function:: vars([object]) - Return the :attr:`__dict__` attribute for a module, class, instance, + Return the :attr:`~object.__dict__` attribute for a module, class, instance, or any other object with a :attr:`__dict__` attribute. Objects such as modules and instances have an updateable :attr:`__dict__` diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 80a2d925ceb..f02f09d7d04 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -132,8 +132,8 @@ The :mod:`gc` module provides the following functions: Return a list of objects directly referred to by any of the arguments. The referents returned are those objects visited by the arguments' C-level - :attr:`tp_traverse` methods (if any), and may not be all objects actually - directly reachable. :attr:`tp_traverse` methods are supported only by objects + :c:member:`~PyTypeObject.tp_traverse` methods (if any), and may not be all objects actually + directly reachable. :c:member:`~PyTypeObject.tp_traverse` methods are supported only by objects that support garbage collection, and are only required to visit objects that may be involved in a cycle. So, for example, if an integer is directly reachable from an argument, that integer object may or may not appear in the result list. diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index ce11eec1cc2..e26fe286c40 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -33,12 +33,12 @@ The module defines the following items: given a non-trivial value. The new class instance is based on *fileobj*, which can be a regular file, a - :class:`StringIO` object, or any other object which simulates a file. It + :class:`~StringIO.StringIO` object, or any other object which simulates a file. It defaults to ``None``, in which case *filename* is opened to provide a file object. When *fileobj* is not ``None``, the *filename* argument is only used to be - included in the :program:`gzip` file header, which may includes the original + included in the :program:`gzip` file header, which may include the original filename of the uncompressed file. It defaults to the filename of *fileobj*, if discernible; otherwise, it defaults to the empty string, and in this case the original filename is not included in the header. @@ -65,9 +65,9 @@ The module defines the following items: Calling a :class:`GzipFile` object's :meth:`close` method does not close *fileobj*, since you might wish to append more material after the compressed - data. This also allows you to pass a :class:`StringIO` object opened for + data. This also allows you to pass a :class:`~StringIO.StringIO` object opened for writing as *fileobj*, and retrieve the resulting memory buffer using the - :class:`StringIO` object's :meth:`getvalue` method. + :class:`StringIO` object's :meth:`~StringIO.StringIO.getvalue` method. :class:`GzipFile` supports iteration and the :keyword:`with` statement. @@ -77,6 +77,9 @@ The module defines the following items: .. versionchanged:: 2.7 Support for zero-padded files was added. + .. versionadded:: 2.7 + The *mtime* argument. + .. function:: open(filename[, mode[, compresslevel]]) diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index 063ad59716b..aa8e6e81ff5 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -25,12 +25,14 @@ digest are interchangeable. Older algorithms were called message digests. The modern term is secure hash. .. note:: - If you want the adler32 or crc32 hash functions they are available in + + If you want the adler32 or crc32 hash functions, they are available in the :mod:`zlib` module. .. warning:: - Some algorithms have known hash collision weaknesses, see the FAQ at the end. + Some algorithms have known hash collision weaknesses, refer to the "See + also" section at the end. There is one constructor method named for each type of :dfn:`hash`. All return a hash object with the same simple interface. For example: use :func:`sha1` to diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst index f0723b7b275..e8acd6c2e0a 100644 --- a/Doc/library/heapq.rst +++ b/Doc/library/heapq.rst @@ -260,7 +260,7 @@ A nice feature of this sort is that you can efficiently insert new items while the sort is going on, provided that the inserted items are not "better" than the last 0'th element you extracted. This is especially useful in simulation contexts, where the tree holds all incoming events, and the "win" condition -means the smallest scheduled time. When an event schedule other events for +means the smallest scheduled time. When an event schedules other events for execution, they are scheduled into the future, so they can easily go into the heap. So, a heap is a good structure for implementing schedulers (this is what I used for my MIDI sequencer :-). diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index e962ff0de71..f88a03e0f97 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -20,7 +20,7 @@ This module implements the HMAC algorithm as described by :rfc:`2104`. Return a new hmac object. If *msg* is present, the method call ``update(msg)`` is made. *digestmod* is the digest constructor or module for the HMAC object to - use. It defaults to the :func:`hashlib.md5` constructor. + use. It defaults to the :data:`hashlib.md5` constructor. An HMAC object has the following methods: diff --git a/Doc/library/httplib.rst b/Doc/library/httplib.rst index 472fa61e13d..fcdfbc015d4 100644 --- a/Doc/library/httplib.rst +++ b/Doc/library/httplib.rst @@ -626,6 +626,6 @@ be created via ``PUT`` request. Here is an example session that shows how to do >>> conn = httplib.HTTPConnection("localhost", 8080) >>> conn.request("PUT", "/file", BODY) >>> response = conn.getresponse() - >>> print resp.status, response.reason + >>> print response.status, response.reason 200, OK diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index b40e4701536..36d78b0991d 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -33,8 +33,8 @@ Menus File menu ^^^^^^^^^ -New window - create a new editing window +New file + create a new file editing window Open... open an existing file diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 899408687b7..cc6cc95c86c 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -296,6 +296,9 @@ I/O Base Classes to control the number of lines read: no more lines will be read if the total size (in bytes/characters) of all lines so far exceeds *hint*. + Note that it's already possible to iterate on file objects using ``for + line in file: ...`` without calling ``file.readlines()``. + .. method:: seek(offset, whence=SEEK_SET) Change the stream position to the given byte *offset*. *offset* is diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 0e98c1e2adf..76452811cde 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -732,9 +732,9 @@ which incur interpreter overhead. next(b, None) return izip(a, b) - def grouper(n, iterable, fillvalue=None): + def grouper(iterable, n, fillvalue=None): "Collect data into fixed-length chunks or blocks" - # grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx + # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx args = [iter(iterable)] * n return izip_longest(fillvalue=fillvalue, *args) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 6e1a9c3d2ed..caee9531dba 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -348,6 +348,8 @@ Encoders and Decoders those with character codes in the 0-31 range, including ``'\t'`` (tab), ``'\n'``, ``'\r'`` and ``'\0'``. + If the data being deserialized is not a valid JSON document, a + :exc:`ValueError` will be raised. .. method:: decode(s) diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 80326123d21..2dc82acf3f6 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -17,6 +17,10 @@ * :ref:`Advanced Tutorial ` * :ref:`Logging Cookbook ` +**Source code:** :source:`Lib/logging/config.py` + +-------------- + This section describes the API for configuring the logging module. .. _logging-config-api: @@ -104,8 +108,9 @@ in :mod:`logging` itself) and defining handlers which are declared either in configurations. If no port is specified, the module's default :const:`DEFAULT_LOGGING_CONFIG_PORT` is used. Logging configurations will be sent as a file suitable for processing by :func:`fileConfig`. Returns a - :class:`Thread` instance on which you can call :meth:`start` to start the - server, and which you can :meth:`join` when appropriate. To stop the server, + :class:`~threading.Thread` instance on which you can call + :meth:`~threading.Thread.start` to start the server, and which you can + :meth:`~threading.Thread.join` when appropriate. To stop the server, call :func:`stopListening`. To send a configuration to the socket, read in the configuration file and @@ -169,11 +174,11 @@ otherwise, the context is used to determine what to instantiate. * *formatters* - the corresponding value will be a dict in which each key is a formatter id and each value is a dict describing how to - configure the corresponding Formatter instance. + configure the corresponding :class:`~logging.Formatter` instance. The configuring dict is searched for keys ``format`` and ``datefmt`` (with defaults of ``None``) and these are used to construct a - :class:`logging.Formatter` instance. + :class:`~logging.Formatter` instance. * *filters* - the corresponding value will be a dict in which each key is a filter id and each value is a dict describing how to configure @@ -711,8 +716,9 @@ format string, with a comma separator. An example time in ISO8601 format is The ``class`` entry is optional. It indicates the name of the formatter's class (as a dotted module and class name.) This option is useful for instantiating a -:class:`Formatter` subclass. Subclasses of :class:`Formatter` can present -exception tracebacks in an expanded or condensed format. +:class:`~logging.Formatter` subclass. Subclasses of +:class:`~logging.Formatter` can present exception tracebacks in an expanded or +condensed format. .. note:: Due to the use of :func:`eval` as described above, there are potential security risks which result from using the :func:`listen` to send diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index a771ca1ce3f..3c05e9ae13d 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -17,6 +17,10 @@ * :ref:`Advanced Tutorial ` * :ref:`Logging Cookbook ` +**Source code:** :source:`Lib/logging/handlers.py` + +-------------- + .. currentmodule:: logging The following useful handlers are provided in the package. Note that three of @@ -53,8 +57,8 @@ and :meth:`flush` methods). .. method:: flush() Flushes the stream by calling its :meth:`flush` method. Note that the - :meth:`close` method is inherited from :class:`Handler` and so does - no output, so an explicit :meth:`flush` call may be needed at times. + :meth:`close` method is inherited from :class:`~logging.Handler` and so + does no output, so an explicit :meth:`flush` call may be needed at times. .. _file-handler: @@ -142,8 +146,8 @@ new stream. This handler is not appropriate for use under Windows, because under Windows open log files cannot be moved or renamed - logging opens the files with exclusive locks - and so there is no need for such a handler. Furthermore, -*ST_INO* is not supported under Windows; :func:`stat` always returns zero for -this value. +*ST_INO* is not supported under Windows; :func:`~os.stat` always returns zero +for this value. .. class:: WatchedFileHandler(filename[,mode[, encoding[, delay]]]) @@ -305,7 +309,8 @@ sends logging output to a network socket. The base class uses a TCP socket. binary format. If there is an error with the socket, silently drops the packet. If the connection was previously lost, re-establishes the connection. To unpickle the record at the receiving end into a - :class:`LogRecord`, use the :func:`makeLogRecord` function. + :class:`~logging.LogRecord`, use the :func:`~logging.makeLogRecord` + function. .. method:: handleError() @@ -383,7 +388,8 @@ over UDP sockets. Pickles the record's attribute dictionary and writes it to the socket in binary format. If there is an error with the socket, silently drops the packet. To unpickle the record at the receiving end into a - :class:`LogRecord`, use the :func:`makeLogRecord` function. + :class:`~logging.LogRecord`, use the :func:`~logging.makeLogRecord` + function. .. method:: makeSocket() diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index a2d903365bf..d3984c78440 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -20,6 +20,9 @@ * :ref:`Advanced Tutorial ` * :ref:`Logging Cookbook ` +**Source code:** :source:`Lib/logging/__init__.py` + +-------------- .. versionadded:: 2.3 diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index 3a87c13ff8a..c01656403ed 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -661,7 +661,7 @@ Maildir, mbox, MH, Babyl, and MMDF. In Babyl mailboxes, the headers of a message are not stored contiguously with the body of the message. To generate a file-like representation, the - headers and body are copied together into a :class:`StringIO` instance + headers and body are copied together into a :class:`~StringIO.StringIO` instance (from the :mod:`StringIO` module), which has an API identical to that of a file. As a result, the file-like object is truly independent of the underlying mailbox but does not save memory compared to a string diff --git a/Doc/library/mimewriter.rst b/Doc/library/mimewriter.rst index 2070ff614d7..a30caef4831 100644 --- a/Doc/library/mimewriter.rst +++ b/Doc/library/mimewriter.rst @@ -24,7 +24,7 @@ to rearrange their order. Return a new instance of the :class:`MimeWriter` class. The only argument passed, *fp*, is a file object to be used for writing. Note that a - :class:`StringIO` object could also be used. + :class:`~StringIO.StringIO` object could also be used. .. _mimewriter-objects: diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 58d00c81cae..ac1963f5bbe 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -152,8 +152,9 @@ memory but does not update the underlying file. .. method:: close() - Close the file. Subsequent calls to other methods of the object will - result in an exception being raised. + Closes the mmap. Subsequent calls to other methods of the object will + result in a ValueError exception being raised. This will not close + the open file. .. method:: find(string[, start[, end]]) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index b7d11941c73..86e8d24c818 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -287,6 +287,9 @@ For example:: print result.get(timeout=1) # prints "100" unless your computer is *very* slow print pool.map(f, range(10)) # prints "[0, 1, 4,..., 81]" +Note that the methods of a pool should only ever be used by the +process which created it. + Reference --------- @@ -379,7 +382,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the Unix daemons or services, they are normal processes that will be terminated (and not joined) if non-daemonic processes have exited. - In addition to the :class:`Threading.Thread` API, :class:`Process` objects + In addition to the :class:`threading.Thread` API, :class:`Process` objects also support the following attributes and methods: .. attribute:: pid @@ -398,7 +401,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the The process's authentication key (a byte string). When :mod:`multiprocessing` is initialized the main process is assigned a - random string using :func:`os.random`. + random string using :func:`os.urandom`. When a :class:`Process` object is created, it will inherit the authentication key of its parent process, although this may be changed by @@ -423,9 +426,9 @@ The :mod:`multiprocessing` package mostly replicates the API of the acquired a lock or semaphore etc. then terminating it is liable to cause other processes to deadlock. - Note that the :meth:`start`, :meth:`join`, :meth:`is_alive` and - :attr:`exit_code` methods should only be called by the process that created - the process object. + Note that the :meth:`start`, :meth:`join`, :meth:`is_alive`, + :meth:`terminate` and :attr:`exitcode` methods should only be called by + the process that created the process object. Example usage of some of the methods of :class:`Process`: @@ -486,6 +489,24 @@ Note that one can also create a shared queue by using a manager object -- see the :mod:`multiprocessing` namespace so you need to import them from :mod:`Queue`. +.. note:: + + When an object is put on a queue, the object is pickled and a + background thread later flushes the pickled data to an underlying + pipe. This has some consequences which are a little surprising, + but should not cause any practical difficulties -- if they really + bother you then you can instead use a queue created with a + :ref:`manager `. + + (1) After putting an object on an empty queue there may be an + infinitesimal delay before the queue's :meth:`~Queue.empty` + method returns :const:`False` and :meth:`~Queue.get_nowait` can + return without raising :exc:`Queue.Empty`. + + (2) If multiple processes are enqueuing objects, it is possible for + the objects to be received at the other end out-of-order. + However, objects enqueued by the same process will always be in + the expected order with respect to each other. .. warning:: @@ -497,7 +518,8 @@ Note that one can also create a shared queue by using a manager object -- see .. warning:: As mentioned above, if a child process has put items on a queue (and it has - not used :meth:`JoinableQueue.cancel_join_thread`), then that process will + not used :meth:`JoinableQueue.cancel_join_thread + `), then that process will not terminate until all buffered items have been flushed to the pipe. This means that if you try joining that process you may get a deadlock unless @@ -609,6 +631,13 @@ For an example of the usage of queues for interprocess communication see the background thread from being joined automatically when the process exits -- see :meth:`join_thread`. + A better name for this method might be + ``allow_exit_without_flush()``. It is likely to cause enqueued + data to lost, and you almost certainly will not need to use it. + It is really only there if you need the current process to exit + immediately without waiting to flush enqueued data to the + underlying pipe, and you don't care about lost data. + .. class:: multiprocessing.queues.SimpleQueue() @@ -639,7 +668,7 @@ For an example of the usage of queues for interprocess communication see call to :meth:`task_done` tells the queue that the processing on the task is complete. - If a :meth:`~Queue.join` is currently blocking, it will resume when all + If a :meth:`~Queue.Queue.join` is currently blocking, it will resume when all items have been processed (meaning that a :meth:`task_done` call was received for every item that had been :meth:`~Queue.put` into the queue). @@ -655,7 +684,7 @@ For an example of the usage of queues for interprocess communication see queue. The count goes down whenever a consumer thread calls :meth:`task_done` to indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, - :meth:`~Queue.join` unblocks. + :meth:`~Queue.Queue.join` unblocks. Miscellaneous @@ -1021,8 +1050,9 @@ processes. array. If *lock* is ``True`` (the default) then a new lock object is created to - synchronize access to the value. If *lock* is a :class:`Lock` or - :class:`RLock` object then that will be used to synchronize access to the + synchronize access to the value. If *lock* is a + :class:`~multiprocessing.Lock` or :class:`~multiprocessing.RLock` object + then that will be used to synchronize access to the value. If *lock* is ``False`` then access to the returned object will not be automatically protected by a lock, so it will not necessarily be "process-safe". @@ -1036,8 +1066,8 @@ processes. object. If *lock* is ``True`` (the default) then a new lock object is created to - synchronize access to the value. If *lock* is a :class:`Lock` or - :class:`RLock` object then that will be used to synchronize access to the + synchronize access to the value. If *lock* is a :class:`~multiprocessing.Lock` or + :class:`~multiprocessing.RLock` object then that will be used to synchronize access to the value. If *lock* is ``False`` then access to the returned object will not be automatically protected by a lock, so it will not necessarily be "process-safe". @@ -1219,8 +1249,8 @@ their parent process exits. The manager classes are defined in the :attr:`proxytype._exposed_` is used instead if it exists.) In the case where no exposed list is specified, all "public methods" of the shared object will be accessible. (Here a "public method" means any attribute - which has a :meth:`__call__` method and whose name does not begin with - ``'_'``.) + which has a :meth:`~object.__call__` method and whose name does not begin + with ``'_'``.) *method_to_typeid* is a mapping used to specify the return type of those exposed methods which should return a proxy. It maps method names to @@ -1581,6 +1611,9 @@ with the :class:`Pool` class. *initializer* is not ``None`` then each worker process will call ``initializer(*initargs)`` when it starts. + Note that the methods of the pool object should only be called by + the process which created the pool. + .. versionadded:: 2.7 *maxtasksperchild* is the number of tasks a worker process can complete before it will exit and be replaced with a fresh worker process, to enable @@ -1727,7 +1760,8 @@ Listeners and Clients :synopsis: API for dealing with sockets. Usually message passing between processes is done using queues or by using -:class:`Connection` objects returned by :func:`Pipe`. +:class:`~multiprocessing.Connection` objects returned by +:func:`~multiprocessing.Pipe`. However, the :mod:`multiprocessing.connection` module allows some extra flexibility. It basically gives a high level message oriented API for dealing @@ -1744,7 +1778,7 @@ authentication* using the :mod:`hmac` module. then a welcome message is sent to the other end of the connection. Otherwise :exc:`AuthenticationError` is raised. -.. function:: answerChallenge(connection, authkey) +.. function:: answer_challenge(connection, authkey) Receive a message, calculate the digest of the message using *authkey* as the key, and then send the digest back. @@ -1793,7 +1827,8 @@ authentication* using the :mod:`hmac` module. private temporary directory created using :func:`tempfile.mkstemp`. If the listener object uses a socket then *backlog* (1 by default) is passed - to the :meth:`listen` method of the socket once it has been bound. + to the :meth:`~socket.socket.listen` method of the socket once it has been + bound. If *authenticate* is ``True`` (``False`` by default) or *authkey* is not ``None`` then digest authentication is used. @@ -1810,8 +1845,9 @@ authentication* using the :mod:`hmac` module. .. method:: accept() Accept a connection on the bound socket or named pipe of the listener - object and return a :class:`Connection` object. If authentication is - attempted and fails, then :exc:`AuthenticationError` is raised. + object and return a :class:`~multiprocessing.Connection` object. If + authentication is attempted and fails, then + :exc:`~multiprocessing.AuthenticationError` is raised. .. method:: close() @@ -1907,7 +1943,8 @@ an ``'AF_PIPE'`` address rather than an ``'AF_UNIX'`` address. Authentication keys ~~~~~~~~~~~~~~~~~~~ -When one uses :meth:`Connection.recv`, the data received is automatically +When one uses :meth:`Connection.recv `, the +data received is automatically unpickled. Unfortunately unpickling data from an untrusted source is a security risk. Therefore :class:`Listener` and :func:`Client` use the :mod:`hmac` module to provide digest authentication. @@ -2056,9 +2093,10 @@ Joining zombie processes On Unix when a process finishes but has not been joined it becomes a zombie. There should never be very many because each time a new process starts (or - :func:`active_children` is called) all completed processes which have not - yet been joined will be joined. Also calling a finished process's - :meth:`Process.is_alive` will join the process. Even so it is probably good + :func:`~multiprocessing.active_children` is called) all completed processes + which have not yet been joined will be joined. Also calling a finished + process's :meth:`Process.is_alive ` will + join the process. Even so it is probably good practice to explicitly join all the processes that you start. Better to inherit than pickle/unpickle @@ -2071,13 +2109,15 @@ Better to inherit than pickle/unpickle Avoid terminating processes - Using the :meth:`Process.terminate` method to stop a process is liable to + Using the :meth:`Process.terminate ` + method to stop a process is liable to cause any shared resources (such as locks, semaphores, pipes and queues) currently being used by the process to become broken or unavailable to other processes. Therefore it is probably best to only consider using - :meth:`Process.terminate` on processes which never use any shared resources. + :meth:`Process.terminate ` on processes + which never use any shared resources. Joining processes that use queues @@ -2161,7 +2201,7 @@ Beware of replacing :data:`sys.stdin` with a "file like object" resulting in a bad file descriptor error, but introduces a potential danger to applications which replace :func:`sys.stdin` with a "file-like object" with output buffering. This danger is that if multiple processes call - :func:`close()` on this file-like object, it could result in the same + :meth:`~io.IOBase.close()` on this file-like object, it could result in the same data being flushed to the object multiple times, resulting in corruption. If you write a file-like object and implement your own caching, you can @@ -2190,14 +2230,16 @@ More picklability as the ``target`` argument on Windows --- just define a function and use that instead. - Also, if you subclass :class:`Process` then make sure that instances will be - picklable when the :meth:`Process.start` method is called. + Also, if you subclass :class:`~multiprocessing.Process` then make sure that + instances will be picklable when the :meth:`Process.start + ` method is called. Global variables Bear in mind that if code run in a child process tries to access a global variable, then the value it sees (if any) may not be the same as the value - in the parent process at the time that :meth:`Process.start` was called. + in the parent process at the time that :meth:`Process.start + ` was called. However, global variables which are just module level constants cause no problems. @@ -2252,7 +2294,7 @@ Demonstration of how to create and use customized managers and proxies: .. literalinclude:: ../includes/mp_newtype.py -Using :class:`Pool`: +Using :class:`~multiprocessing.pool.Pool`: .. literalinclude:: ../includes/mp_pool.py diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index 323fd696c2d..713c8dfc41c 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -25,6 +25,14 @@ the Unix :program:`ftp` program and other FTP clients. no argument is given, the file :file:`.netrc` in the user's home directory will be read. Parse errors will raise :exc:`NetrcParseError` with diagnostic information including the file name, line number, and terminating token. + If no argument is specified on a POSIX system, the presence of passwords in + the :file:`.netrc` file will raise a :exc:`NetrcParseError` if the file + ownership or permissions are insecure (owned by a user other than the user + running the process, or accessible for read or write by any other user). + This implements security behavior equivalent to that of ftp and other + programs that use :file:`.netrc`. + + .. versionchanged:: 2.7.6 Added the POSIX permissions check. .. exception:: NetrcParseError diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index f46e8ac824e..8811b5df8ce 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -73,10 +73,10 @@ The numeric tower .. class:: Integral - Subtypes :class:`Rational` and adds a conversion to :class:`int`. - Provides defaults for :func:`float`, :attr:`~Rational.numerator`, and - :attr:`~Rational.denominator`, and bit-string operations: ``<<``, - ``>>``, ``&``, ``^``, ``|``, ``~``. + Subtypes :class:`Rational` and adds a conversion to :class:`int`. Provides + defaults for :func:`float`, :attr:`~Rational.numerator`, and + :attr:`~Rational.denominator`. Adds abstract methods for ``**`` and + bit-string operations: ``<<``, ``>>``, ``&``, ``^``, ``|``, ``~``. Notes for type implementors diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index 53d45b2306e..244da144f21 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -490,13 +490,22 @@ lookups. These are useful for making fast field extractors as arguments for expect a function argument. -.. function:: attrgetter(attr[, args...]) +.. function:: attrgetter(attr) + attrgetter(*attrs) - Return a callable object that fetches *attr* from its operand. If more than one - attribute is requested, returns a tuple of attributes. After, - ``f = attrgetter('name')``, the call ``f(b)`` returns ``b.name``. After, - ``f = attrgetter('name', 'date')``, the call ``f(b)`` returns ``(b.name, - b.date)``. Equivalent to:: + Return a callable object that fetches *attr* from its operand. + If more than one attribute is requested, returns a tuple of attributes. + The attribute names can also contain dots. For example: + + * After ``f = attrgetter('name')``, the call ``f(b)`` returns ``b.name``. + + * After ``f = attrgetter('name', 'date')``, the call ``f(b)`` returns + ``(b.name, b.date)``. + + * After ``f = attrgetter('name.first', 'name.last')``, the call ``f(b)`` + returns ``(r.name.first, r.name.last)``. + + Equivalent to:: def attrgetter(*items): if len(items) == 1: @@ -505,7 +514,7 @@ expect a function argument. return resolve_attr(obj, attr) else: def g(obj): - return tuple(resolve_att(obj, attr) for attr in items) + return tuple(resolve_attr(obj, attr) for attr in items) return g def resolve_attr(obj, attr): @@ -514,9 +523,6 @@ expect a function argument. return obj - The attribute names can also contain dots; after ``f = attrgetter('date.month')``, - the call ``f(b)`` returns ``b.date.month``. - .. versionadded:: 2.4 .. versionchanged:: 2.5 @@ -526,11 +532,19 @@ expect a function argument. Added support for dotted attributes. -.. function:: itemgetter(item[, args...]) +.. function:: itemgetter(item) + itemgetter(*items) Return a callable object that fetches *item* from its operand using the operand's :meth:`__getitem__` method. If multiple items are specified, - returns a tuple of lookup values. Equivalent to:: + returns a tuple of lookup values. For example: + + * After ``f = itemgetter(2)``, the call ``f(r)`` returns ``r[2]``. + + * After ``g = itemgetter(2, 5, 3)``, the call ``g(r)`` returns + ``(r[2], r[5], r[3])``. + + Equivalent to:: def itemgetter(*items): if len(items) == 1: @@ -573,9 +587,14 @@ expect a function argument. Return a callable object that calls the method *name* on its operand. If additional arguments and/or keyword arguments are given, they will be given - to the method as well. After ``f = methodcaller('name')``, the call ``f(b)`` - returns ``b.name()``. After ``f = methodcaller('name', 'foo', bar=1)``, the - call ``f(b)`` returns ``b.name('foo', bar=1)``. Equivalent to:: + to the method as well. For example: + + * After ``f = methodcaller('name')``, the call ``f(b)`` returns ``b.name()``. + + * After ``f = methodcaller('name', 'foo', bar=1)``, the call ``f(b)`` + returns ``b.name('foo', bar=1)``. + + Equivalent to:: def methodcaller(name, *args, **kwargs): def caller(obj): diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 5e245d0df5b..e9b98132e8a 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -147,7 +147,7 @@ the :mod:`glob` module.) .. function:: getctime(path) Return the system's ctime which, on some systems (like Unix) is the time of the - last change, and, on others (like Windows), is the creation time for *path*. + last metadata change, and, on others (like Windows), is the creation time for *path*. The return value is a number giving the number of seconds since the epoch (see the :mod:`time` module). Raise :exc:`os.error` if the file does not exist or is inaccessible. @@ -236,8 +236,10 @@ the :mod:`glob` module.) .. function:: relpath(path[, start]) - Return a relative filepath to *path* either from the current directory or from - an optional *start* point. + Return a relative filepath to *path* either from the current directory or + from an optional *start* directory. This is a path computation: the + filesystem is not accessed to confirm the existence or nature of *path* or + *start*. *start* defaults to :attr:`os.curdir`. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 106c1df35c0..3d48722d1cf 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -737,8 +737,6 @@ as internal buffering of data. Return ``True`` if the file descriptor *fd* is open and connected to a tty(-like) device, else ``False``. - Availability: Unix. - .. function:: lseek(fd, pos, how) @@ -1388,8 +1386,8 @@ Files and Directories On some Unix systems (such as Linux), the following attributes may also be available: - * :attr:`st_blocks` - number of blocks allocated for file - * :attr:`st_blksize` - filesystem blocksize + * :attr:`st_blocks` - number of 512-byte blocks allocated for file + * :attr:`st_blksize` - filesystem blocksize for efficient file system I/O * :attr:`st_rdev` - type of device if an inode device * :attr:`st_flags` - user defined flags for file @@ -2503,8 +2501,9 @@ Miscellaneous Functions This function returns random bytes from an OS-specific randomness source. The returned data should be unpredictable enough for cryptographic applications, though its exact quality depends on the OS implementation. On a UNIX-like - system this will query /dev/urandom, and on Windows it will use CryptGenRandom. - If a randomness source is not found, :exc:`NotImplementedError` will be raised. + system this will query ``/dev/urandom``, and on Windows it will use + ``CryptGenRandom()``. If a randomness source is not found, + :exc:`NotImplementedError` will be raised. For an easy-to-use interface to the random number generator provided by your platform, please see :class:`random.SystemRandom`. diff --git a/Doc/library/ossaudiodev.rst b/Doc/library/ossaudiodev.rst index c5404deb9c7..1a65d246a8c 100644 --- a/Doc/library/ossaudiodev.rst +++ b/Doc/library/ossaudiodev.rst @@ -296,7 +296,7 @@ simple calculations. fmt = dsp.setfmt(fmt) channels = dsp.channels(channels) - rate = dsp.rate(channels) + rate = dsp.rate(rate) .. method:: oss_audio_device.bufsize() diff --git a/Doc/library/othergui.rst b/Doc/library/othergui.rst index 69df9df85eb..28b57717648 100644 --- a/Doc/library/othergui.rst +++ b/Doc/library/othergui.rst @@ -14,7 +14,7 @@ available for Python: the C one. It comes with many more widgets than Tkinter provides, and has good Python-specific reference documentation. There are also bindings to `GNOME `_. One well known PyGTK application is - `PythonCAD `_. An online `tutorial + `WingIDE `_. An online `tutorial `_ is available. `PyQt `_ diff --git a/Doc/library/pipes.rst b/Doc/library/pipes.rst index 6d7dff8c378..415d5c766b9 100644 --- a/Doc/library/pipes.rst +++ b/Doc/library/pipes.rst @@ -35,7 +35,7 @@ Example:: .. function:: quote(s) - .. deprecated:: 1.6 + .. deprecated:: 2.7 Prior to Python 2.7, this function was not publicly documented. It is finally exposed publicly in Python 3.3 as the :func:`quote ` function in the :mod:`shlex` module. diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 26f587eaf3c..cb7144a64ad 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -197,8 +197,8 @@ Windows Platform .. function:: win32_ver(release='', version='', csd='', ptype='') Get additional version information from the Windows Registry and return a tuple - ``(version, csd, ptype)`` referring to version number, CSD level - (service pack) and OS type (multi/single processor). + ``(release, version, csd, ptype)`` referring to OS release, version number, + CSD level (service pack) and OS type (multi/single processor). As a hint: *ptype* is ``'Uniprocessor Free'`` on single processor NT machines and ``'Multiprocessor Free'`` on multi processor machines. The *'Free'* refers diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 627f2156ab9..0fb14892ff1 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -4,11 +4,6 @@ The Python Profilers ******************** -.. sectionauthor:: James Roskind - -.. module:: profile - :synopsis: Python source profiler. - **Source code:** :source:`Lib/profile.py` and :source:`Lib/pstats.py` -------------- @@ -22,34 +17,31 @@ Introduction to the profilers single: deterministic profiling single: profiling, deterministic -A :dfn:`profiler` is a program that describes the run time performance -of a program, providing a variety of statistics. This documentation -describes the profiler functionality provided in the modules -:mod:`cProfile`, :mod:`profile` and :mod:`pstats`. This profiler -provides :dfn:`deterministic profiling` of Python programs. It also -provides a series of report generation tools to allow users to rapidly -examine the results of a profile operation. +:mod:`cProfile` and :mod:`profile` provide :dfn:`deterministic profiling` of +Python programs. A :dfn:`profile` is a set of statistics that describes how +often and for how long various parts of the program executed. These statistics +can be formatted into reports via the :mod:`pstats` module. -The Python standard library provides three different profilers: +The Python standard library provides three different implementations of the same +profiling interface: -#. :mod:`cProfile` is recommended for most users; it's a C extension - with reasonable overhead - that makes it suitable for profiling long-running programs. - Based on :mod:`lsprof`, - contributed by Brett Rosen and Ted Czotter. +1. :mod:`cProfile` is recommended for most users; it's a C extension with + reasonable overhead that makes it suitable for profiling long-running + programs. Based on :mod:`lsprof`, contributed by Brett Rosen and Ted + Czotter. .. versionadded:: 2.5 -#. :mod:`profile`, a pure Python module whose interface is imitated by - :mod:`cProfile`. Adds significant overhead to profiled programs. - If you're trying to extend - the profiler in some way, the task might be easier with this module. +2. :mod:`profile`, a pure Python module whose interface is imitated by + :mod:`cProfile`, but which adds significant overhead to profiled programs. + If you're trying to extend the profiler in some way, the task might be easier + with this module. .. versionchanged:: 2.4 Now also reports the time spent in calls to built-in functions and methods. -#. :mod:`hotshot` was an experimental C module that focused on minimizing +3. :mod:`hotshot` was an experimental C module that focused on minimizing the overhead of profiling, at the expense of longer data post-processing times. It is no longer maintained and may be dropped in a future version of Python. @@ -66,6 +58,15 @@ is newer and might not be available on all systems. :mod:`_lsprof` module. The :mod:`hotshot` module is reserved for specialized usage. +.. note:: + + The profiler modules are designed to provide an execution profile for a given + program, not for benchmarking purposes (for that, there is :mod:`timeit` for + reasonably accurate results). This particularly applies to benchmarking + Python code against C code: the profilers introduce overhead for Python code, + but not for C-level functions, and so the C code would seem faster than any + Python one. + .. _profile-instant: @@ -76,57 +77,94 @@ This section is provided for users that "don't want to read the manual." It provides a very brief overview, and allows a user to rapidly perform profiling on an existing application. -To profile an application with a main entry point of :func:`foo`, you would add -the following to your module:: +To profile a function that takes a single argument, you can do:: import cProfile - cProfile.run('foo()') + import re + cProfile.run('re.compile("foo|bar")') (Use :mod:`profile` instead of :mod:`cProfile` if the latter is not available on your system.) -The above action would cause :func:`foo` to be run, and a series of informative -lines (the profile) to be printed. The above approach is most useful when -working with the interpreter. If you would like to save the results of a -profile into a file for later examination, you can supply a file name as the -second argument to the :func:`run` function:: +The above action would run :func:`re.compile` and print profile results like +the following:: - import cProfile - cProfile.run('foo()', 'fooprof') + 197 function calls (192 primitive calls) in 0.002 seconds -The file :file:`cProfile.py` can also be invoked as a script to profile another -script. For example:: + Ordered by: standard name - python -m cProfile myscript.py + ncalls tottime percall cumtime percall filename:lineno(function) + 1 0.000 0.000 0.001 0.001 :1() + 1 0.000 0.000 0.001 0.001 re.py:212(compile) + 1 0.000 0.000 0.001 0.001 re.py:268(_compile) + 1 0.000 0.000 0.000 0.000 sre_compile.py:172(_compile_charset) + 1 0.000 0.000 0.000 0.000 sre_compile.py:201(_optimize_charset) + 4 0.000 0.000 0.000 0.000 sre_compile.py:25(_identityfunction) + 3/1 0.000 0.000 0.000 0.000 sre_compile.py:33(_compile) -:file:`cProfile.py` accepts two optional arguments on the command line:: +The first line indicates that 197 calls were monitored. Of those calls, 192 +were :dfn:`primitive`, meaning that the call was not induced via recursion. The +next line: ``Ordered by: standard name``, indicates that the text string in the +far right column was used to sort the output. The column headings include: - cProfile.py [-o output_file] [-s sort_order] +ncalls + for the number of calls, -``-s`` only applies to standard output (``-o`` is not supplied). -Look in the :class:`Stats` documentation for valid sort values. +tottime + for the total time spent in the given function (and excluding time made in + calls to sub-functions) -When you wish to review the profile, you should use the methods in the -:mod:`pstats` module. Typically you would load the statistics data as follows:: +percall + is the quotient of ``tottime`` divided by ``ncalls`` - import pstats - p = pstats.Stats('fooprof') +cumtime + is the cumulative time spent in this and all subfunctions (from invocation + till exit). This figure is accurate *even* for recursive functions. -The class :class:`Stats` (the above code just created an instance of this class) -has a variety of methods for manipulating and printing the data that was just -read into ``p``. When you ran :func:`cProfile.run` above, what was printed was -the result of three method calls:: +percall + is the quotient of ``cumtime`` divided by primitive calls - p.strip_dirs().sort_stats(-1).print_stats() +filename:lineno(function) + provides the respective data of each function + +When there are two numbers in the first column (for example ``3/1``), it means +that the function recursed. The second value is the number of primitive calls +and the former is the total number of calls. Note that when the function does +not recurse, these two values are the same, and only the single figure is +printed. -The first method removed the extraneous path from all the module names. The -second method sorted all the entries according to the standard module/line/name -string that is printed. The third method printed out all the statistics. You -might try the following sort calls: +Instead of printing the output at the end of the profile run, you can save the +results to a file by specifying a filename to the :func:`run` function:: -.. (this is to comply with the semantics of the old profiler). + import cProfile + import re + cProfile.run('re.compile("foo|bar")', 'restats') + +The :class:`pstats.Stats` class reads profile results from a file and formats +them in various ways. + +The file :mod:`cProfile` can also be invoked as a script to profile another +script. For example:: + + python -m cProfile [-o output_file] [-s sort_order] myscript.py + +``-o`` writes the profile results to a file instead of to stdout -:: +``-s`` specifies one of the :func:`~pstats.Stats.sort_stats` sort values to sort +the output by. This only applies when ``-o`` is not supplied. + +The :mod:`pstats` module's :class:`~pstats.Stats` class has a variety of methods +for manipulating and printing the data saved into a profile results file:: + + import pstats + p = pstats.Stats('restats') + p.strip_dirs().sort_stats(-1).print_stats() + +The :meth:`~pstats.Stats.strip_dirs` method removed the extraneous path from all +the module names. The :meth:`~pstats.Stats.sort_stats` method sorted all the +entries according to the standard module/line/name string that is printed. The +:meth:`~pstats.Stats.print_stats` method printed out all the statistics. You +might try the following sort calls:: p.sort_stats('name') p.print_stats() @@ -175,343 +213,340 @@ If you want more functionality, you're going to have to read the manual, or guess what the following functions do:: p.print_callees() - p.add('fooprof') + p.add('restats') Invoked as a script, the :mod:`pstats` module is a statistics browser for reading and examining profile dumps. It has a simple line-oriented interface (implemented using :mod:`cmd`) and interactive help. +:mod:`profile` and :mod:`cProfile` Module Reference +======================================================= -.. _deterministic-profiling: +.. module:: cProfile +.. module:: profile + :synopsis: Python source profiler. -What Is Deterministic Profiling? -================================ +Both the :mod:`profile` and :mod:`cProfile` modules provide the following +functions: -:dfn:`Deterministic profiling` is meant to reflect the fact that all *function -call*, *function return*, and *exception* events are monitored, and precise -timings are made for the intervals between these events (during which time the -user's code is executing). In contrast, :dfn:`statistical profiling` (which is -not done by this module) randomly samples the effective instruction pointer, and -deduces where time is being spent. The latter technique traditionally involves -less overhead (as the code does not need to be instrumented), but provides only -relative indications of where time is being spent. +.. function:: run(command, filename=None, sort=-1) -In Python, since there is an interpreter active during execution, the presence -of instrumented code is not required to do deterministic profiling. Python -automatically provides a :dfn:`hook` (optional callback) for each event. In -addition, the interpreted nature of Python tends to add so much overhead to -execution, that deterministic profiling tends to only add small processing -overhead in typical applications. The result is that deterministic profiling is -not that expensive, yet provides extensive run time statistics about the -execution of a Python program. + This function takes a single argument that can be passed to the :func:`exec` + function, and an optional file name. In all cases this routine executes:: -Call count statistics can be used to identify bugs in code (surprising counts), -and to identify possible inline-expansion points (high call counts). Internal -time statistics can be used to identify "hot loops" that should be carefully -optimized. Cumulative time statistics should be used to identify high level -errors in the selection of algorithms. Note that the unusual handling of -cumulative times in this profiler allows statistics for recursive -implementations of algorithms to be directly compared to iterative -implementations. + exec(command, __main__.__dict__, __main__.__dict__) + and gathers profiling statistics from the execution. If no file name is + present, then this function automatically creates a :class:`~pstats.Stats` + instance and prints a simple profiling report. If the sort value is specified + it is passed to this :class:`~pstats.Stats` instance to control how the + results are sorted. -Reference Manual -- :mod:`profile` and :mod:`cProfile` -====================================================== +.. function:: runctx(command, globals, locals, filename=None) -.. module:: cProfile - :synopsis: Python profiler + This function is similar to :func:`run`, with added arguments to supply the + globals and locals dictionaries for the *command* string. This routine + executes:: + exec(command, globals, locals) -The primary entry point for the profiler is the global function -:func:`profile.run` (resp. :func:`cProfile.run`). It is typically used to create -any profile information. The reports are formatted and printed using methods of -the class :class:`pstats.Stats`. The following is a description of all of these -standard entry points and functions. For a more in-depth view of some of the -code, consider reading the later section on Profiler Extensions, which includes -discussion of how to derive "better" profilers from the classes presented, or -reading the source code for these modules. + and gathers profiling statistics as in the :func:`run` function above. +.. class:: Profile(timer=None, timeunit=0.0, subcalls=True, builtins=True) -.. function:: run(command[, filename][, sort]) + This class is normally only used if more precise control over profiling is + needed than what the :func:`cProfile.run` function provides. - This function takes a single argument that can be passed to the - :keyword:`exec` statement, and optionally a file name and a sorting - directive. In all cases this routine attempts to :keyword:`exec` - its first argument, and gather profiling statistics from the - execution. If no file name is present, then this function - automatically prints a simple profiling report, sorted by the - standard name string (file/line/function-name) that is presented in - each line. The following is a typical output from such a call:: + A custom timer can be supplied for measuring how long code takes to run via + the *timer* argument. This must be a function that returns a single number + representing the current time. If the number is an integer, the *timeunit* + specifies a multiplier that specifies the duration of each unit of time. For + example, if the timer returns times measured in thousands of seconds, the + time unit would be ``.001``. - 2706 function calls (2004 primitive calls) in 4.504 CPU seconds + Directly using the :class:`Profile` class allows formatting profile results + without writing the profile data to a file:: - Ordered by: standard name + import cProfile, pstats, StringIO + pr = cProfile.Profile() + pr.enable() + # ... do something ... + pr.disable() + s = StringIO.StringIO() + sortby = 'cumulative' + ps = pstats.Stats(pr, stream=s).sort_stats(sortby) + ps.print_stats() + print s.getvalue() - ncalls tottime percall cumtime percall filename:lineno(function) - 2 0.006 0.003 0.953 0.477 pobject.py:75(save_objects) - 43/3 0.533 0.012 0.749 0.250 pobject.py:99(evaluate) - ... + .. method:: enable() - The first line indicates that 2706 calls were monitored. Of those - calls, 2004 were :dfn:`primitive`. We define :dfn:`primitive` to - mean that the call was not induced via recursion. The next line: - ``Ordered by: standard name``, indicates that the text string in - the far right column was used to sort the output. The column - headings include: + Start collecting profiling data. - ncalls - for the number of calls, + .. method:: disable() - tottime - for the total time spent in the given function (and - excluding time made in calls to sub-functions), + Stop collecting profiling data. - percall - is the quotient of ``tottime`` divided by ``ncalls`` + .. method:: create_stats() - cumtime - is the total time spent in this and all subfunctions (from invocation till - exit). This figure is accurate *even* for recursive functions. + Stop collecting profiling data and record the results internally + as the current profile. - percall - is the quotient of ``cumtime`` divided by primitive calls + .. method:: print_stats(sort=-1) - filename:lineno(function) - provides the respective data of each function + Create a :class:`~pstats.Stats` object based on the current + profile and print the results to stdout. - When there are two numbers in the first column (for example, - ``43/3``), then the latter is the number of primitive calls, and - the former is the actual number of calls. Note that when the - function does not recurse, these two values are the same, and only - the single figure is printed. For information on the sort - argument, refer to :meth:`pstats.Stats.sort_stats`. + .. method:: dump_stats(filename) + Write the results of the current profile to *filename*. -.. function:: runctx(command, globals, locals[, filename]) + .. method:: run(cmd) - This function is similar to :func:`run`, with added arguments to supply the - globals and locals dictionaries for the *command* string. + Profile the cmd via :func:`exec`. -Analysis of the profiler data is done using the :class:`Stats` class. + .. method:: runctx(cmd, globals, locals) -.. note:: + Profile the cmd via :func:`exec` with the specified global and + local environment. + + .. method:: runcall(func, *args, **kwargs) + + Profile ``func(*args, **kwargs)`` + +.. _profile-stats: - The :class:`Stats` class is defined in the :mod:`pstats` module. +The :class:`Stats` Class +======================== +Analysis of the profiler data is done using the :class:`~pstats.Stats` class. .. module:: pstats :synopsis: Statistics object for use with the profiler. +.. class:: Stats(*filenames or profile, stream=sys.stdout) -.. class:: Stats(filename, stream=sys.stdout[, ...]) + This class constructor creates an instance of a "statistics object" from a + *filename* (or list of filenames) or from a :class:`Profile` instance. Output + will be printed to the stream specified by *stream*. - This class constructor creates an instance of a "statistics object" - from a *filename* (or set of filenames). :class:`Stats` objects - are manipulated by methods, in order to print useful reports. You - may specify an alternate output stream by giving the keyword - argument, ``stream``. + The file selected by the above constructor must have been created by the + corresponding version of :mod:`profile` or :mod:`cProfile`. To be specific, + there is *no* file compatibility guaranteed with future versions of this + profiler, and there is no compatibility with files produced by other + profilers. If several files are provided, all the statistics for identical + functions will be coalesced, so that an overall view of several processes can + be considered in a single report. If additional files need to be combined + with data in an existing :class:`~pstats.Stats` object, the + :meth:`~pstats.Stats.add` method can be used. - The file selected by the above constructor must have been created - by the corresponding version of :mod:`profile` or :mod:`cProfile`. - To be specific, there is *no* file compatibility guaranteed with - future versions of this profiler, and there is no compatibility - with files produced by other profilers. If several files are - provided, all the statistics for identical functions will be - coalesced, so that an overall view of several processes can be - considered in a single report. If additional files need to be - combined with data in an existing :class:`Stats` object, the - :meth:`add` method can be used. + Instead of reading the profile data from a file, a :class:`cProfile.Profile` + or :class:`profile.Profile` object can be used as the profile data source. - .. (such as the old system profiler). + :class:`Stats` objects have the following methods: - .. versionchanged:: 2.5 - The *stream* parameter was added. + .. method:: strip_dirs() + This method for the :class:`Stats` class removes all leading path + information from file names. It is very useful in reducing the size of + the printout to fit within (close to) 80 columns. This method modifies + the object, and the stripped information is lost. After performing a + strip operation, the object is considered to have its entries in a + "random" order, as it was just after object initialization and loading. + If :meth:`~pstats.Stats.strip_dirs` causes two function names to be + indistinguishable (they are on the same line of the same filename, and + have the same function name), then the statistics for these two entries + are accumulated into a single entry. -.. _profile-stats: -The :class:`Stats` Class ------------------------- + .. method:: add(*filenames) -:class:`Stats` objects have the following methods: + This method of the :class:`Stats` class accumulates additional profiling + information into the current profiling object. Its arguments should refer + to filenames created by the corresponding version of :func:`profile.run` + or :func:`cProfile.run`. Statistics for identically named (re: file, line, + name) functions are automatically accumulated into single function + statistics. -.. method:: Stats.strip_dirs() + .. method:: dump_stats(filename) - This method for the :class:`Stats` class removes all leading path - information from file names. It is very useful in reducing the - size of the printout to fit within (close to) 80 columns. This - method modifies the object, and the stripped information is lost. - After performing a strip operation, the object is considered to - have its entries in a "random" order, as it was just after object - initialization and loading. If :meth:`strip_dirs` causes two - function names to be indistinguishable (they are on the same line - of the same filename, and have the same function name), then the - statistics for these two entries are accumulated into a single - entry. + Save the data loaded into the :class:`Stats` object to a file named + *filename*. The file is created if it does not exist, and is overwritten + if it already exists. This is equivalent to the method of the same name + on the :class:`profile.Profile` and :class:`cProfile.Profile` classes. + .. versionadded:: 2.3 -.. method:: Stats.add(filename[, ...]) - This method of the :class:`Stats` class accumulates additional profiling - information into the current profiling object. Its arguments should refer to - filenames created by the corresponding version of :func:`profile.run` or - :func:`cProfile.run`. Statistics for identically named (re: file, line, name) - functions are automatically accumulated into single function statistics. + .. method:: sort_stats(*keys) + + This method modifies the :class:`Stats` object by sorting it according to + the supplied criteria. The argument is typically a string identifying the + basis of a sort (example: ``'time'`` or ``'name'``). + + When more than one key is provided, then additional keys are used as + secondary criteria when there is equality in all keys selected before + them. For example, ``sort_stats('name', 'file')`` will sort all the + entries according to their function name, and resolve all ties (identical + function names) by sorting by file name. + + Abbreviations can be used for any key names, as long as the abbreviation + is unambiguous. The following are the keys currently defined: + + +------------------+----------------------+ + | Valid Arg | Meaning | + +==================+======================+ + | ``'calls'`` | call count | + +------------------+----------------------+ + | ``'cumulative'`` | cumulative time | + +------------------+----------------------+ + | ``'cumtime'`` | cumulative time | + +------------------+----------------------+ + | ``'file'`` | file name | + +------------------+----------------------+ + | ``'filename'`` | file name | + +------------------+----------------------+ + | ``'module'`` | file name | + +------------------+----------------------+ + | ``'ncalls'`` | call count | + +------------------+----------------------+ + | ``'pcalls'`` | primitive call count | + +------------------+----------------------+ + | ``'line'`` | line number | + +------------------+----------------------+ + | ``'name'`` | function name | + +------------------+----------------------+ + | ``'nfl'`` | name/file/line | + +------------------+----------------------+ + | ``'stdname'`` | standard name | + +------------------+----------------------+ + | ``'time'`` | internal time | + +------------------+----------------------+ + | ``'tottime'`` | internal time | + +------------------+----------------------+ + + Note that all sorts on statistics are in descending order (placing most + time consuming items first), where as name, file, and line number searches + are in ascending order (alphabetical). The subtle distinction between + ``'nfl'`` and ``'stdname'`` is that the standard name is a sort of the + name as printed, which means that the embedded line numbers get compared + in an odd way. For example, lines 3, 20, and 40 would (if the file names + were the same) appear in the string order 20, 3 and 40. In contrast, + ``'nfl'`` does a numeric compare of the line numbers. In fact, + ``sort_stats('nfl')`` is the same as ``sort_stats('name', 'file', + 'line')``. + + For backward-compatibility reasons, the numeric arguments ``-1``, ``0``, + ``1``, and ``2`` are permitted. They are interpreted as ``'stdname'``, + ``'calls'``, ``'time'``, and ``'cumulative'`` respectively. If this old + style format (numeric) is used, only one sort key (the numeric key) will + be used, and additional arguments will be silently ignored. + + .. For compatibility with the old profiler. + + + .. method:: reverse_order() + + This method for the :class:`Stats` class reverses the ordering of the + basic list within the object. Note that by default ascending vs + descending order is properly selected based on the sort key of choice. + + .. This method is provided primarily for compatibility with the old + profiler. + + + .. method:: print_stats(*restrictions) + + This method for the :class:`Stats` class prints out a report as described + in the :func:`profile.run` definition. + + The order of the printing is based on the last + :meth:`~pstats.Stats.sort_stats` operation done on the object (subject to + caveats in :meth:`~pstats.Stats.add` and + :meth:`~pstats.Stats.strip_dirs`). + + The arguments provided (if any) can be used to limit the list down to the + significant entries. Initially, the list is taken to be the complete set + of profiled functions. Each restriction is either an integer (to select a + count of lines), or a decimal fraction between 0.0 and 1.0 inclusive (to + select a percentage of lines), or a regular expression (to pattern match + the standard name that is printed. If several restrictions are provided, + then they are applied sequentially. For example:: + + print_stats(.1, 'foo:') + + would first limit the printing to first 10% of list, and then only print + functions that were part of filename :file:`.\*foo:`. In contrast, the + command:: + + print_stats('foo:', .1) + + would limit the list to all functions having file names :file:`.\*foo:`, + and then proceed to only print the first 10% of them. + + + .. method:: print_callers(*restrictions) + + This method for the :class:`Stats` class prints a list of all functions + that called each function in the profiled database. The ordering is + identical to that provided by :meth:`~pstats.Stats.print_stats`, and the + definition of the restricting argument is also identical. Each caller is + reported on its own line. The format differs slightly depending on the + profiler that produced the stats: + + * With :mod:`profile`, a number is shown in parentheses after each caller + to show how many times this specific call was made. For convenience, a + second non-parenthesized number repeats the cumulative time spent in the + function at the right. + + * With :mod:`cProfile`, each caller is preceded by three numbers: the + number of times this specific call was made, and the total and + cumulative times spent in the current function while it was invoked by + this specific caller. + + + .. method:: print_callees(*restrictions) + This method for the :class:`Stats` class prints a list of all function + that were called by the indicated function. Aside from this reversal of + direction of calls (re: called vs was called by), the arguments and + ordering are identical to the :meth:`~pstats.Stats.print_callers` method. -.. method:: Stats.dump_stats(filename) - Save the data loaded into the :class:`Stats` object to a file named - *filename*. The file is created if it does not exist, and is - overwritten if it already exists. This is equivalent to the method - of the same name on the :class:`profile.Profile` and - :class:`cProfile.Profile` classes. +.. _deterministic-profiling: - .. versionadded:: 2.3 +What Is Deterministic Profiling? +================================ +:dfn:`Deterministic profiling` is meant to reflect the fact that all *function +call*, *function return*, and *exception* events are monitored, and precise +timings are made for the intervals between these events (during which time the +user's code is executing). In contrast, :dfn:`statistical profiling` (which is +not done by this module) randomly samples the effective instruction pointer, and +deduces where time is being spent. The latter technique traditionally involves +less overhead (as the code does not need to be instrumented), but provides only +relative indications of where time is being spent. -.. method:: Stats.sort_stats(key[, ...]) - - This method modifies the :class:`Stats` object by sorting it - according to the supplied criteria. The argument is typically a - string identifying the basis of a sort (example: ``'time'`` or - ``'name'``). - - When more than one key is provided, then additional keys are used - as secondary criteria when there is equality in all keys selected - before them. For example, ``sort_stats('name', 'file')`` will sort - all the entries according to their function name, and resolve all - ties (identical function names) by sorting by file name. - - Abbreviations can be used for any key names, as long as the abbreviation is - unambiguous. The following are the keys currently defined: - - +------------------+----------------------+ - | Valid Arg | Meaning | - +==================+======================+ - | ``'calls'`` | call count | - +------------------+----------------------+ - | ``'cumulative'`` | cumulative time | - +------------------+----------------------+ - | ``'cumtime'`` | cumulative time | - +------------------+----------------------+ - | ``'file'`` | file name | - +------------------+----------------------+ - | ``'filename'`` | file name | - +------------------+----------------------+ - | ``'module'`` | file name | - +------------------+----------------------+ - | ``'ncalls'`` | call count | - +------------------+----------------------+ - | ``'pcalls'`` | primitive call count | - +------------------+----------------------+ - | ``'line'`` | line number | - +------------------+----------------------+ - | ``'name'`` | function name | - +------------------+----------------------+ - | ``'nfl'`` | name/file/line | - +------------------+----------------------+ - | ``'stdname'`` | standard name | - +------------------+----------------------+ - | ``'time'`` | internal time | - +------------------+----------------------+ - | ``'tottime'`` | internal time | - +------------------+----------------------+ - - Note that all sorts on statistics are in descending order (placing - most time consuming items first), where as name, file, and line - number searches are in ascending order (alphabetical). The subtle - distinction between ``'nfl'`` and ``'stdname'`` is that the - standard name is a sort of the name as printed, which means that - the embedded line numbers get compared in an odd way. For example, - lines 3, 20, and 40 would (if the file names were the same) appear - in the string order 20, 3 and 40. In contrast, ``'nfl'`` does a - numeric compare of the line numbers. In fact, - ``sort_stats('nfl')`` is the same as ``sort_stats('name', 'file', - 'line')``. - - For backward-compatibility reasons, the numeric arguments ``-1``, - ``0``, ``1``, and ``2`` are permitted. They are interpreted as - ``'stdname'``, ``'calls'``, ``'time'``, and ``'cumulative'`` - respectively. If this old style format (numeric) is used, only one - sort key (the numeric key) will be used, and additional arguments - will be silently ignored. - - .. For compatibility with the old profiler, - - -.. method:: Stats.reverse_order() - - This method for the :class:`Stats` class reverses the ordering of - the basic list within the object. Note that by default ascending - vs descending order is properly selected based on the sort key of - choice. - - .. This method is provided primarily for compatibility with the old profiler. - - -.. method:: Stats.print_stats([restriction, ...]) - - This method for the :class:`Stats` class prints out a report as - described in the :func:`profile.run` definition. - - The order of the printing is based on the last :meth:`sort_stats` - operation done on the object (subject to caveats in :meth:`add` and - :meth:`strip_dirs`). - - The arguments provided (if any) can be used to limit the list down - to the significant entries. Initially, the list is taken to be the - complete set of profiled functions. Each restriction is either an - integer (to select a count of lines), or a decimal fraction between - 0.0 and 1.0 inclusive (to select a percentage of lines), or a - regular expression (to pattern match the standard name that is - printed; as of Python 1.5b1, this uses the Perl-style regular - expression syntax defined by the :mod:`re` module). If several - restrictions are provided, then they are applied sequentially. For - example:: - - print_stats(.1, 'foo:') - - would first limit the printing to first 10% of list, and then only print - functions that were part of filename :file:`.\*foo:`. In contrast, the - command:: - - print_stats('foo:', .1) - - would limit the list to all functions having file names :file:`.\*foo:`, and - then proceed to only print the first 10% of them. - - -.. method:: Stats.print_callers([restriction, ...]) - - This method for the :class:`Stats` class prints a list of all functions that - called each function in the profiled database. The ordering is identical to - that provided by :meth:`print_stats`, and the definition of the restricting - argument is also identical. Each caller is reported on its own line. The - format differs slightly depending on the profiler that produced the stats: - - * With :mod:`profile`, a number is shown in parentheses after each caller to - show how many times this specific call was made. For convenience, a second - non-parenthesized number repeats the cumulative time spent in the function - at the right. - - * With :mod:`cProfile`, each caller is preceded by three numbers: - the number of times this specific call was made, and the total - and cumulative times spent in the current function while it was - invoked by this specific caller. - - -.. method:: Stats.print_callees([restriction, ...]) +In Python, since there is an interpreter active during execution, the presence +of instrumented code is not required to do deterministic profiling. Python +automatically provides a :dfn:`hook` (optional callback) for each event. In +addition, the interpreted nature of Python tends to add so much overhead to +execution, that deterministic profiling tends to only add small processing +overhead in typical applications. The result is that deterministic profiling is +not that expensive, yet provides extensive run time statistics about the +execution of a Python program. - This method for the :class:`Stats` class prints a list of all - function that were called by the indicated function. Aside from - this reversal of direction of calls (re: called vs was called by), - the arguments and ordering are identical to the - :meth:`print_callers` method. +Call count statistics can be used to identify bugs in code (surprising counts), +and to identify possible inline-expansion points (high call counts). Internal +time statistics can be used to identify "hot loops" that should be carefully +optimized. Cumulative time statistics should be used to identify high level +errors in the selection of algorithms. Note that the unusual handling of +cumulative times in this profiler allows statistics for recursive +implementations of algorithms to be directly compared to iterative +implementations. -.. _profile-limits: +.. _profile-limitations: Limitations =========== @@ -554,7 +589,7 @@ The profiler of the :mod:`profile` module subtracts a constant from each event handling time to compensate for the overhead of calling the time function, and socking away the results. By default, the constant is 0. The following procedure can be used to obtain a better constant for a given platform (see -discussion in section Limitations above). :: +:ref:`profile-limitations`). :: import profile pr = profile.Profile() @@ -564,8 +599,8 @@ discussion in section Limitations above). :: The method executes the number of Python calls given by the argument, directly and again under the profiler, measuring the time for both. It then computes the hidden overhead per profiler event, and returns that as a float. For example, -on an 800 MHz Pentium running Windows 2000, and using Python's time.clock() as -the timer, the magical number is about 12.5e-6. +on a 1.8Ghz Intel Core i5 running Mac OS X, and using Python's time.clock() as +the timer, the magical number is about 4.04e-6. The object of this exercise is to get a fairly consistent result. If your computer is *very* fast, or your timer function has poor resolution, you might @@ -588,57 +623,50 @@ When you have a consistent answer, there are three ways you can use it: [#]_ :: If you have a choice, you are better off choosing a smaller constant, and then your results will "less often" show up as negative in profile statistics. +.. _profile-timers: -.. _profiler-extensions: - -Extensions --- Deriving Better Profilers -======================================== - -The :class:`Profile` class of both modules, :mod:`profile` and :mod:`cProfile`, -were written so that derived classes could be developed to extend the profiler. -The details are not described here, as doing this successfully requires an -expert understanding of how the :class:`Profile` class works internally. Study -the source code of the module carefully if you want to pursue this. +Using a custom timer +==================== -If all you want to do is change how current time is determined (for example, to -force use of wall-clock time or elapsed process time), pass the timing function -you want to the :class:`Profile` class constructor:: +If you want to change how current time is determined (for example, to force use +of wall-clock time or elapsed process time), pass the timing function you want +to the :class:`Profile` class constructor:: - pr = profile.Profile(your_time_func) + pr = profile.Profile(your_time_func) -The resulting profiler will then call :func:`your_time_func`. +The resulting profiler will then call ``your_time_func``. Depending on whether +you are using :class:`profile.Profile` or :class:`cProfile.Profile`, +``your_time_func``'s return value will be interpreted differently: :class:`profile.Profile` - :func:`your_time_func` should return a single number, or a list of - numbers whose sum is the current time (like what :func:`os.times` - returns). If the function returns a single time number, or the - list of returned numbers has length 2, then you will get an - especially fast version of the dispatch routine. - - Be warned that you should calibrate the profiler class for the - timer function that you choose. For most machines, a timer that - returns a lone integer value will provide the best results in terms - of low overhead during profiling. (:func:`os.times` is *pretty* - bad, as it returns a tuple of floating point values). If you want - to substitute a better timer in the cleanest fashion, derive a - class and hardwire a replacement dispatch method that best handles - your timer call, along with the appropriate calibration constant. + ``your_time_func`` should return a single number, or a list of numbers whose + sum is the current time (like what :func:`os.times` returns). If the + function returns a single time number, or the list of returned numbers has + length 2, then you will get an especially fast version of the dispatch + routine. + + Be warned that you should calibrate the profiler class for the timer function + that you choose (see :ref:`profile-calibration`). For most machines, a timer + that returns a lone integer value will provide the best results in terms of + low overhead during profiling. (:func:`os.times` is *pretty* bad, as it + returns a tuple of floating point values). If you want to substitute a + better timer in the cleanest fashion, derive a class and hardwire a + replacement dispatch method that best handles your timer call, along with the + appropriate calibration constant. :class:`cProfile.Profile` - :func:`your_time_func` should return a single number. If it - returns plain integers, you can also invoke the class constructor - with a second argument specifying the real duration of one unit of - time. For example, if :func:`your_integer_time_func` returns times - measured in thousands of seconds, you would construct the - :class:`Profile` instance as follows:: - - pr = profile.Profile(your_integer_time_func, 0.001) - - As the :mod:`cProfile.Profile` class cannot be calibrated, custom - timer functions should be used with care and should be as fast as - possible. For the best results with a custom timer, it might be - necessary to hard-code it in the C source of the internal - :mod:`_lsprof` module. + ``your_time_func`` should return a single number. If it returns integers, + you can also invoke the class constructor with a second argument specifying + the real duration of one unit of time. For example, if + ``your_integer_time_func`` returns times measured in thousands of seconds, + you would construct the :class:`Profile` instance as follows:: + + pr = cProfile.Profile(your_integer_time_func, 0.001) + + As the :mod:`cProfile.Profile` class cannot be calibrated, custom timer + functions should be used with care and should be as fast as possible. For + the best results with a custom timer, it might be necessary to hard-code it + in the C source of the internal :mod:`_lsprof` module. .. rubric:: Footnotes diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index 179c4c67c69..b52570550da 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -60,13 +60,15 @@ The :mod:`Queue` module defines the following classes and exceptions: .. exception:: Empty - Exception raised when non-blocking :meth:`get` (or :meth:`get_nowait`) is called + Exception raised when non-blocking :meth:`~Queue.get` (or + :meth:`~Queue.get_nowait`) is called on a :class:`Queue` object which is empty. .. exception:: Full - Exception raised when non-blocking :meth:`put` (or :meth:`put_nowait`) is called + Exception raised when non-blocking :meth:`~Queue.put` (or + :meth:`~Queue.put_nowait`) is called on a :class:`Queue` object which is full. .. seealso:: diff --git a/Doc/library/random.rst b/Doc/library/random.rst index b1df231e736..1bc998940dc 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -60,6 +60,13 @@ The :mod:`random` module also provides the :class:`SystemRandom` class which uses the system function :func:`os.urandom` to generate random numbers from sources provided by the operating system. +.. warning:: + + The pseudo-random generators of this module should not be used for + security purposes. Use :func:`os.urandom` or :class:`SystemRandom` if + you require a cryptographically secure pseudo-random number generator. + + Bookkeeping functions: diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 1af4cf2011d..91afbd497c9 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -237,21 +237,32 @@ The special characters are: ``(?P...)`` Similar to regular parentheses, but the substring matched by the group is - accessible within the rest of the regular expression via the symbolic group - name *name*. Group names must be valid Python identifiers, and each group - name must be defined only once within a regular expression. A symbolic group - is also a numbered group, just as if the group were not named. So the group - named ``id`` in the example below can also be referenced as the numbered group - ``1``. - - For example, if the pattern is ``(?P[a-zA-Z_]\w*)``, the group can be - referenced by its name in arguments to methods of match objects, such as - ``m.group('id')`` or ``m.end('id')``, and also by name in the regular - expression itself (using ``(?P=id)``) and replacement text given to - ``.sub()`` (using ``\g``). + accessible via the symbolic group name *name*. Group names must be valid + Python identifiers, and each group name must be defined only once within a + regular expression. A symbolic group is also a numbered group, just as if + the group were not named. + + Named groups can be referenced in three contexts. If the pattern is + ``(?P['"]).*?(?P=quote)`` (i.e. matching a string quoted with either + single or double quotes): + + +---------------------------------------+----------------------------------+ + | Context of reference to group "quote" | Ways to reference it | + +=======================================+==================================+ + | in the same pattern itself | * ``(?P=quote)`` (as shown) | + | | * ``\1`` | + +---------------------------------------+----------------------------------+ + | when processing match object ``m`` | * ``m.group('quote')`` | + | | * ``m.end('quote')`` (etc.) | + +---------------------------------------+----------------------------------+ + | in a string passed to the ``repl`` | * ``\g`` | + | argument of ``re.sub()`` | * ``\g<1>`` | + | | * ``\1`` | + +---------------------------------------+----------------------------------+ ``(?P=name)`` - Matches whatever text was matched by the earlier group named *name*. + A backreference to a named group; it matches whatever text was matched by the + earlier group named *name*. ``(?#...)`` A comment; the contents of the parentheses are simply ignored. @@ -311,7 +322,7 @@ the second character. For example, ``\$`` matches the character ``'$'``. ``\number`` Matches the contents of the group of the same number. Groups are numbered starting from 1. For example, ``(.+) \1`` matches ``'the the'`` or ``'55 55'``, - but not ``'the end'`` (note the space after the group). This special sequence + but not ``'thethe'`` (note the space after the group). This special sequence can only be used to match one of the first 99 groups. If the first digit of *number* is 0, or *number* is 3 octal digits long, it will not be interpreted as a group match, but as the character with octal value *number*. Inside the @@ -331,7 +342,8 @@ the second character. For example, ``\$`` matches the character ``'$'``. depends on the values of the ``UNICODE`` and ``LOCALE`` flags. For example, ``r'\bfoo\b'`` matches ``'foo'``, ``'foo.'``, ``'(foo)'``, ``'bar foo baz'`` but not ``'foobar'`` or ``'foo3'``. - Inside a character range, ``\b`` represents the backspace character, for compatibility with Python's string literals. + Inside a character range, ``\b`` represents the backspace character, for + compatibility with Python's string literals. ``\B`` Matches the empty string, but only when it is *not* at the beginning or end of a @@ -642,7 +654,8 @@ form. when not adjacent to a previous match, so ``sub('x*', '-', 'abc')`` returns ``'-a-b-c-'``. - In addition to character escapes and backreferences as described above, + In string-type *repl* arguments, in addition to the character escapes and + backreferences described above, ``\g`` will use the substring matched by the group named ``name``, as defined by the ``(?P...)`` syntax. ``\g`` uses the corresponding group number; ``\g<2>`` is therefore equivalent to ``\2``, but isn't ambiguous diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index 834daceb8cc..7ca45347f11 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -42,6 +42,11 @@ which cannot be checked or controlled by the operating system are not defined in this module for those platforms. +.. data:: RLIM_INFINITY + + Constant used to represent the the limit for an unlimited resource. + + .. function:: getrlimit(resource) Returns a tuple ``(soft, hard)`` with the current soft and hard limits of @@ -53,12 +58,20 @@ this module for those platforms. Sets new limits of consumption of *resource*. The *limits* argument must be a tuple ``(soft, hard)`` of two integers describing the new limits. A value of - ``-1`` can be used to specify the maximum possible upper limit. + :data:`~resource.RLIM_INFINITY` can be used to request a limit that is + unlimited. Raises :exc:`ValueError` if an invalid resource is specified, if the new soft - limit exceeds the hard limit, or if a process tries to raise its hard limit - (unless the process has an effective UID of super-user). Can also raise - :exc:`error` if the underlying system call fails. + limit exceeds the hard limit, or if a process tries to raise its hard limit. + Specifying a limit of :data:`~resource.RLIM_INFINITY` when the hard or + system limit for that resource is not unlimited will result in a + :exc:`ValueError`. A process with the effective UID of super-user can + request any valid limit value, including unlimited, but :exc:`ValueError` + will still be raised if the requested limit exceeds the system imposed + limit. + + ``setrlimit`` may also raise :exc:`error` if the underlying system call + fails. These symbols define resources whose consumption can be controlled using the :func:`setrlimit` and :func:`getrlimit` functions described below. The values of diff --git a/Doc/library/select.rst b/Doc/library/select.rst index d131cb98fba..24cb7567c88 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -63,7 +63,7 @@ The module defines the following: This is a straightforward interface to the Unix :c:func:`select` system call. The first three arguments are sequences of 'waitable objects': either integers representing file descriptors or objects with a parameterless method - named :meth:`fileno` returning such an integer: + named :meth:`~io.IOBase.fileno` returning such an integer: * *rlist*: wait until ready for reading * *wlist*: wait until ready for writing @@ -88,8 +88,8 @@ The module defines the following: Among the acceptable object types in the sequences are Python file objects (e.g. ``sys.stdin``, or objects returned by :func:`open` or :func:`os.popen`), socket objects returned by :func:`socket.socket`. You may also define a :dfn:`wrapper` - class yourself, as long as it has an appropriate :meth:`fileno` method (that - really returns a file descriptor, not just a random integer). + class yourself, as long as it has an appropriate :meth:`~io.IOBase.fileno` + method (that really returns a file descriptor, not just a random integer). .. note:: @@ -207,10 +207,10 @@ linearly scanned again. :c:func:`select` is O(highest file descriptor), while .. method:: poll.register(fd[, eventmask]) Register a file descriptor with the polling object. Future calls to the - :meth:`poll` method will then check whether the file descriptor has any pending - I/O events. *fd* can be either an integer, or an object with a :meth:`fileno` - method that returns an integer. File objects implement :meth:`fileno`, so they - can also be used as the argument. + :meth:`poll` method will then check whether the file descriptor has any + pending I/O events. *fd* can be either an integer, or an object with a + :meth:`~io.IOBase.fileno` method that returns an integer. File objects + implement :meth:`!fileno`, so they can also be used as the argument. *eventmask* is an optional bitmask describing the type of events you want to check for, and can be a combination of the constants :const:`POLLIN`, @@ -251,7 +251,7 @@ linearly scanned again. :c:func:`select` is O(highest file descriptor), while Remove a file descriptor being tracked by a polling object. Just like the :meth:`register` method, *fd* can be an integer or an object with a - :meth:`fileno` method that returns an integer. + :meth:`~io.IOBase.fileno` method that returns an integer. Attempting to remove a file descriptor that was never registered causes a :exc:`KeyError` exception to be raised. diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index ad36b458969..b02f763dc36 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -47,9 +47,11 @@ lots of shared sub-objects. The keys are ordinary strings. Like file objects, shelve objects should be closed explicitly to ensure that the persistent data is flushed to disk. - Since the :mod:`shelve` module stores objects using :mod:`pickle`, the same - security precautions apply. Accordingly, you should avoid loading a shelf - from an untrusted source. +.. warning:: + + Because the :mod:`shelve` module is backed by :mod:`pickle`, it is insecure + to load a shelf from an untrusted source. Like with pickle, loading a shelf + can execute arbitrary code. Shelf objects support all methods supported by dictionaries. This eases the transition from dictionary based scripts to those requiring persistent storage. diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index 044bbde28b8..e145659f62e 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -24,9 +24,12 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). A :class:`SMTP` instance encapsulates an SMTP connection. It has methods that support a full repertoire of SMTP and ESMTP operations. If the optional - host and port parameters are given, the SMTP :meth:`connect` method is called - with those parameters during initialization. An :exc:`SMTPConnectError` is - raised if the specified host doesn't respond correctly. The optional + host and port parameters are given, the SMTP :meth:`connect` method is + called with those parameters during initialization. If specified, + *local_hostname* is used as the FQDN of the local host in the HELO/EHLO + command. Otherwise, the local hostname is found using + :func:`socket.getfqdn`. If the :meth:`connect` call returns anything other + than a success code, an :exc:`SMTPConnectError` is raised. The optional *timeout* parameter specifies a timeout in seconds for blocking operations like the connection attempt (if not specified, the global default timeout setting will be used). @@ -45,12 +48,13 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). :class:`SMTP`. :class:`SMTP_SSL` should be used for situations where SSL is required from the beginning of the connection and using :meth:`starttls` is not appropriate. If *host* is not specified, the local host is used. If - *port* is omitted, the standard SMTP-over-SSL port (465) is used. *keyfile* - and *certfile* are also optional, and can contain a PEM formatted private key - and certificate chain file for the SSL connection. The optional *timeout* - parameter specifies a timeout in seconds for blocking operations like the - connection attempt (if not specified, the global default timeout setting - will be used). + *port* is omitted, the standard SMTP-over-SSL port (465) is used. + *local_hostname* has the same meaning as it does for the :class:`SMTP` + class. *keyfile* and *certfile* are also optional, and can contain a PEM + formatted private key and certificate chain file for the SSL connection. The + optional *timeout* parameter specifies a timeout in seconds for blocking + operations like the connection attempt (if not specified, the global default + timeout setting will be used). .. versionadded:: 2.6 @@ -58,13 +62,15 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). .. class:: LMTP([host[, port[, local_hostname]]]) The LMTP protocol, which is very similar to ESMTP, is heavily based on the - standard SMTP client. It's common to use Unix sockets for LMTP, so our :meth:`connect` - method must support that as well as a regular host:port server. To specify a - Unix socket, you must use an absolute path for *host*, starting with a '/'. + standard SMTP client. It's common to use Unix sockets for LMTP, so our + :meth:`connect` method must support that as well as a regular host:port + server. *local_hostname* has the same meaning as it does for the + :class:`SMTP` class. To specify a Unix socket, you must use an absolute + path for *host*, starting with a '/'. - Authentication is supported, using the regular SMTP mechanism. When using a Unix - socket, LMTP generally don't support or require any authentication, but your - mileage might vary. + Authentication is supported, using the regular SMTP mechanism. When using a + Unix socket, LMTP generally don't support or require any authentication, but + your mileage might vary. .. versionadded:: 2.6 @@ -73,7 +79,8 @@ A nice selection of exceptions is defined as well: .. exception:: SMTPException - Base exception class for all exceptions raised by this module. + The base exception class for all the other exceptions provided by this + module. .. exception:: SMTPServerDisconnected @@ -152,15 +159,6 @@ An :class:`SMTP` instance has the following methods: for connection and for all messages sent to and received from the server. -.. method:: SMTP.connect([host[, port]]) - - Connect to a host on a given port. The defaults are to connect to the local - host at the standard SMTP port (25). If the hostname ends with a colon (``':'``) - followed by a number, that suffix will be stripped off and the number - interpreted as the port number to use. This method is automatically invoked by - the constructor if a host is specified during instantiation. - - .. method:: SMTP.docmd(cmd, [, argstring]) Send a command *cmd* to the server. The optional argument *argstring* is simply @@ -177,6 +175,17 @@ An :class:`SMTP` instance has the following methods: :exc:`SMTPServerDisconnected` will be raised. +.. method:: SMTP.connect([host[, port]]) + + Connect to a host on a given port. The defaults are to connect to the local + host at the standard SMTP port (25). If the hostname ends with a colon (``':'``) + followed by a number, that suffix will be stripped off and the number + interpreted as the port number to use. This method is automatically invoked by + the constructor if a host is specified during instantiation. Returns a + 2-tuple of the response code and message sent by the server in its + connection response. + + .. method:: SMTP.helo([hostname]) Identify yourself to the SMTP server using ``HELO``. The hostname argument diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index d6ac36431fb..453c9447441 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -28,7 +28,7 @@ want to refer to :rfc:`3493` titled Basic Socket Interface Extensions for IPv6. The Python interface is a straightforward transliteration of the Unix system call and library interface for sockets to Python's object-oriented style: the -:func:`socket` function returns a :dfn:`socket object` whose methods implement +:func:`.socket` function returns a :dfn:`socket object` whose methods implement the various socket system calls. Parameter types are somewhat higher-level than in the C interface: as with :meth:`read` and :meth:`write` operations on Python files, buffer allocation on receive operations is automatic, and buffer length @@ -146,7 +146,7 @@ The module :mod:`socket` exports the following constants and functions: AF_INET6 These constants represent the address (and protocol) families, used for the - first argument to :func:`socket`. If the :const:`AF_UNIX` constant is not + first argument to :func:`.socket`. If the :const:`AF_UNIX` constant is not defined then this protocol is unsupported. @@ -252,7 +252,7 @@ The module :mod:`socket` exports the following constants and functions: ``(family, socktype, proto, canonname, sockaddr)`` In these tuples, *family*, *socktype*, *proto* are all integers and are - meant to be passed to the :func:`socket` function. *canonname* will be + meant to be passed to the :func:`.socket` function. *canonname* will be a string representing the canonical name of the *host* if :const:`AI_CANONNAME` is part of the *flags* argument; else *canonname* will be empty. *sockaddr* is a tuple describing a socket address, whose @@ -343,7 +343,7 @@ The module :mod:`socket` exports the following constants and functions: .. function:: getprotobyname(protocolname) Translate an Internet protocol name (for example, ``'icmp'``) to a constant - suitable for passing as the (optional) third argument to the :func:`socket` + suitable for passing as the (optional) third argument to the :func:`.socket` function. This is usually only needed for sockets opened in "raw" mode (:const:`SOCK_RAW`); for the normal socket modes, the correct protocol is chosen automatically if the protocol is omitted or zero. @@ -377,7 +377,7 @@ The module :mod:`socket` exports the following constants and functions: Build a pair of connected socket objects using the given address family, socket type, and protocol number. Address family, socket type, and protocol number are - as for the :func:`socket` function above. The default family is :const:`AF_UNIX` + as for the :func:`.socket` function above. The default family is :const:`AF_UNIX` if defined on the platform; otherwise, the default is :const:`AF_INET`. Availability: Unix. @@ -388,7 +388,7 @@ The module :mod:`socket` exports the following constants and functions: Duplicate the file descriptor *fd* (an integer as returned by a file object's :meth:`fileno` method) and build a socket object from the result. Address - family, socket type and protocol number are as for the :func:`socket` function + family, socket type and protocol number are as for the :func:`.socket` function above. The file descriptor should refer to a socket, but this is not checked --- subsequent operations on the object may fail if the file descriptor is invalid. This function is rarely needed, but can be used to get or set socket options on @@ -861,10 +861,10 @@ Example Here are four minimal example programs using the TCP/IP protocol: a server that echoes all data that it receives back (servicing only one client), and a client -using it. Note that a server must perform the sequence :func:`socket`, +using it. Note that a server must perform the sequence :func:`.socket`, :meth:`~socket.bind`, :meth:`~socket.listen`, :meth:`~socket.accept` (possibly repeating the :meth:`~socket.accept` to service more than one client), while a -client only needs the sequence :func:`socket`, :meth:`~socket.connect`. Also +client only needs the sequence :func:`.socket`, :meth:`~socket.connect`. Also note that the server does not :meth:`~socket.sendall`/:meth:`~socket.recv` on the socket it is listening on but on the new socket returned by :meth:`~socket.accept`. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index c32bb907114..6ca50d8a553 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -179,7 +179,7 @@ Module functions and constants For the *isolation_level* parameter, please see the :attr:`Connection.isolation_level` property of :class:`Connection` objects. - SQLite natively supports only the types TEXT, INTEGER, FLOAT, BLOB and NULL. If + SQLite natively supports only the types TEXT, INTEGER, REAL, BLOB and NULL. If you want to use other types you must add support for them yourself. The *detect_types* parameter and the using custom **converters** registered with the module-level :func:`register_converter` function allow you to easily do that. diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 19b90a15cb7..550c249894a 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -298,21 +298,37 @@ Functions, Constants, and Exceptions SSLSocket Objects ----------------- -.. method:: SSLSocket.read([nbytes=1024]) - - Reads up to ``nbytes`` bytes from the SSL-encrypted channel and returns them. - -.. method:: SSLSocket.write(data) - - Writes the ``data`` to the other side of the connection, using the SSL - channel to encrypt. Returns the number of bytes written. +SSL sockets provide the following methods of :ref:`socket-objects`: + +- :meth:`~socket.socket.accept()` +- :meth:`~socket.socket.bind()` +- :meth:`~socket.socket.close()` +- :meth:`~socket.socket.connect()` +- :meth:`~socket.socket.fileno()` +- :meth:`~socket.socket.getpeername()`, :meth:`~socket.socket.getsockname()` +- :meth:`~socket.socket.getsockopt()`, :meth:`~socket.socket.setsockopt()` +- :meth:`~socket.socket.gettimeout()`, :meth:`~socket.socket.settimeout()`, + :meth:`~socket.socket.setblocking()` +- :meth:`~socket.socket.listen()` +- :meth:`~socket.socket.makefile()` +- :meth:`~socket.socket.recv()`, :meth:`~socket.socket.recv_into()` + (but passing a non-zero ``flags`` argument is not allowed) +- :meth:`~socket.socket.send()`, :meth:`~socket.socket.sendall()` (with + the same limitation) +- :meth:`~socket.socket.shutdown()` + +However, since the SSL (and TLS) protocol has its own framing atop +of TCP, the SSL sockets abstraction can, in certain respects, diverge from +the specification of normal, OS-level sockets. + +SSL sockets also have the following additional methods and attributes: .. method:: SSLSocket.getpeercert(binary_form=False) If there is no certificate for the peer on the other end of the connection, returns ``None``. - If the parameter ``binary_form`` is :const:`False`, and a certificate was + If the ``binary_form`` parameter is :const:`False`, and a certificate was received from the peer, this method returns a :class:`dict` instance. If the certificate was not validated, the dict is empty. If the certificate was validated, it returns a dict with the keys ``subject`` (the principal for @@ -338,10 +354,16 @@ SSLSocket Objects If the ``binary_form`` parameter is :const:`True`, and a certificate was provided, this method returns the DER-encoded form of the entire certificate as a sequence of bytes, or :const:`None` if the peer did not provide a - certificate. This return value is independent of validation; if validation - was required (:const:`CERT_OPTIONAL` or :const:`CERT_REQUIRED`), it will have - been validated, but if :const:`CERT_NONE` was used to establish the - connection, the certificate, if present, will not have been validated. + certificate. Whether the peer provides a certificate depends on the SSL + socket's role: + + * for a client SSL socket, the server will always provide a certificate, + regardless of whether validation was required; + + * for a server SSL socket, the client will only provide a certificate + when requested by the server; therefore :meth:`getpeercert` will return + :const:`None` if you used :const:`CERT_NONE` (rather than + :const:`CERT_OPTIONAL` or :const:`CERT_REQUIRED`). .. method:: SSLSocket.cipher() @@ -619,10 +641,10 @@ And go back to listening for new client connections. .. seealso:: Class :class:`socket.socket` - Documentation of underlying :mod:`socket` class + Documentation of underlying :mod:`socket` class - `TLS (Transport Layer Security) and SSL (Secure Socket Layer) `_ - Debby Koren + `SSL/TLS Strong Encryption: An Introduction `_ + Intro from the Apache webserver documentation `RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management `_ Steve Kent diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 2761410f0a1..cd7b683769d 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -26,7 +26,7 @@ instances and exceptions. Some operations are supported by several object types; in particular, practically all objects can be compared, tested for truth value, and converted -to a string (with the :func:`repr` function or the slightly different +to a string (with the :ref:`repr() ` function or the slightly different :func:`str` function). The latter function is implicitly used when an object is written by the :func:`print` function. @@ -614,7 +614,7 @@ support: iterators for those iteration types. (An example of an object supporting multiple forms of iteration would be a tree structure which supports both breadth-first and depth-first traversal.) This method corresponds to the - :attr:`tp_iter` slot of the type structure for Python objects in the Python/C + :c:member:`~PyTypeObject.tp_iter` slot of the type structure for Python objects in the Python/C API. The iterator objects themselves are required to support the following two @@ -625,7 +625,7 @@ methods, which together form the :dfn:`iterator protocol`: Return the iterator object itself. This is required to allow both containers and iterators to be used with the :keyword:`for` and :keyword:`in` statements. - This method corresponds to the :attr:`tp_iter` slot of the type structure for + This method corresponds to the :c:member:`~PyTypeObject.tp_iter` slot of the type structure for Python objects in the Python/C API. @@ -633,7 +633,7 @@ methods, which together form the :dfn:`iterator protocol`: Return the next item from the container. If there are no further items, raise the :exc:`StopIteration` exception. This method corresponds to the - :attr:`tp_iternext` slot of the type structure for Python objects in the + :c:member:`~PyTypeObject.tp_iternext` slot of the type structure for Python objects in the Python/C API. Python defines several iterator objects to support iteration over general and @@ -743,10 +743,10 @@ are sequences of the same type; *n*, *i* and *j* are integers: +------------------+--------------------------------+----------+ | ``max(s)`` | largest item of *s* | | +------------------+--------------------------------+----------+ -| ``s.index(i)`` | index of the first occurence | | +| ``s.index(i)`` | index of the first occurrence | | | | of *i* in *s* | | +------------------+--------------------------------+----------+ -| ``s.count(i)`` | total number of occurences of | | +| ``s.count(i)`` | total number of occurrences of | | | | *i* in *s* | | +------------------+--------------------------------+----------+ @@ -931,10 +931,22 @@ string functions based on regular expressions. .. method:: str.expandtabs([tabsize]) Return a copy of the string where all tab characters are replaced by one or - more spaces, depending on the current column and the given tab size. The - column number is reset to zero after each newline occurring in the string. - If *tabsize* is not given, a tab size of ``8`` characters is assumed. This - doesn't understand other non-printing characters or escape sequences. + more spaces, depending on the current column and the given tab size. Tab + positions occur every *tabsize* characters (default is 8, giving tab + positions at columns 0, 8, 16 and so on). To expand the string, the current + column is set to zero and the string is examined character by character. If + the character is a tab (``\t``), one or more space characters are inserted + in the result until the current column is equal to the next tab position. + (The tab character itself is not copied.) If the character is a newline + (``\n``) or return (``\r``), it is copied and the current column is reset to + zero. Any other character is copied unchanged and the current column is + incremented by one regardless of how the character is represented when + printed. + + >>> '01\t012\t0123\t01234'.expandtabs() + '01 012 0123 01234' + >>> '01\t012\t0123\t01234'.expandtabs(4) + '01 012 0123 01234' .. method:: str.find(sub[, start[, end]]) @@ -1452,7 +1464,7 @@ The conversion types are: | | character string). | | +------------+-----------------------------------------------------+-------+ | ``'r'`` | String (converts any Python object using | \(5) | -| | :func:`repr`). | | +| | :ref:`repr() `). | | +------------+-----------------------------------------------------+-------+ | ``'s'`` | String (converts any Python object using | \(6) | | | :func:`str`). | | @@ -1722,11 +1734,11 @@ other sequence-like behavior. There are currently two built-in set types, :class:`set` and :class:`frozenset`. The :class:`set` type is mutable --- the contents can be changed using methods -like :meth:`add` and :meth:`remove`. Since it is mutable, it has no hash value -and cannot be used as either a dictionary key or as an element of another set. -The :class:`frozenset` type is immutable and :term:`hashable` --- its contents -cannot be altered after it is created; it can therefore be used as a dictionary -key or as an element of another set. +like :meth:`~set.add` and :meth:`~set.remove`. Since it is mutable, it has no +hash value and cannot be used as either a dictionary key or as an element of +another set. The :class:`frozenset` type is immutable and :term:`hashable` --- +its contents cannot be altered after it is created; it can therefore be used as +a dictionary key or as an element of another set. As of Python 2.7, non-empty sets (not frozensets) can be created by placing a comma-separated list of elements within braces, for example: ``{'jack', @@ -1738,9 +1750,10 @@ The constructors for both classes work the same: frozenset([iterable]) Return a new set or frozenset object whose elements are taken from - *iterable*. The elements of a set must be hashable. To represent sets of - sets, the inner sets must be :class:`frozenset` objects. If *iterable* is - not specified, a new empty set is returned. + *iterable*. The elements of a set must be :term:`hashable`. To + represent sets of sets, the inner sets must be :class:`frozenset` + objects. If *iterable* is not specified, a new empty set is + returned. Instances of :class:`set` and :class:`frozenset` provide the following operations: @@ -1836,8 +1849,8 @@ The constructors for both classes work the same: based on their members. For example, ``set('abc') == frozenset('abc')`` returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. - The subset and equality comparisons do not generalize to a complete ordering - function. For example, any two disjoint sets are not equal and are not + The subset and equality comparisons do not generalize to a total ordering + function. For example, any two non-empty disjoint sets are not equal and are not subsets of each other, so *all* of the following return ``False``: ``ab``. Accordingly, sets do not implement the :meth:`__cmp__` method. @@ -2815,12 +2828,12 @@ statement is not, strictly speaking, an operation on a module object; ``import foo`` does not require a module object named *foo* to exist, rather it requires an (external) *definition* for a module named *foo* somewhere.) -A special attribute of every module is :attr:`__dict__`. This is the dictionary -containing the module's symbol table. Modifying this dictionary will actually -change the module's symbol table, but direct assignment to the :attr:`__dict__` -attribute is not possible (you can write ``m.__dict__['a'] = 1``, which defines -``m.a`` to be ``1``, but you can't write ``m.__dict__ = {}``). Modifying -:attr:`__dict__` directly is not recommended. +A special attribute of every module is :attr:`~object.__dict__`. This is the +dictionary containing the module's symbol table. Modifying this dictionary will +actually change the module's symbol table, but direct assignment to the +:attr:`__dict__` attribute is not possible (you can write +``m.__dict__['a'] = 1``, which defines ``m.a`` to be ``1``, but you can't write +``m.__dict__ = {}``). Modifying :attr:`__dict__` directly is not recommended. Modules built into the interpreter are written like this: ````. If loaded from a file, they are written as ```_ Book by Mark Rozerman about building attractive and modern graphical user interfaces with Python and Tkinter. - `An Introduction to Tkinter `_ - Fredrik Lundh's on-line reference material. - `Python and Tkinter Programming `_ The book by John Grayson (ISBN 1-884777-81-3). diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index abdef988add..999b024e696 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -91,7 +91,7 @@ need to derive from a specific class. Third-party unittest frameworks with a lighter-weight syntax for writing tests. For example, ``assert func(10) == 42``. - `The Python Testing Tools Taxonomy `_ + `The Python Testing Tools Taxonomy `_ An extensive list of Python testing tools including functional testing frameworks and mock object libraries. @@ -916,8 +916,8 @@ Test cases | :meth:`assertRaises(exc, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | | | ` | | | +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertRaisesRegexp(exc, re, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | 2.7 | - | ` | and the message matches *re* | | + | :meth:`assertRaisesRegexp(exc, r, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | 2.7 | + | ` | and the message matches regex *r* | | +---------------------------------------------------------+--------------------------------------+------------+ .. method:: assertRaises(exception, callable, *args, **kwds) @@ -958,7 +958,7 @@ Test cases a regular expression object or a string containing a regular expression suitable for use by :func:`re.search`. Examples:: - self.assertRaisesRegexp(ValueError, 'invalid literal for.*XYZ$', + self.assertRaisesRegexp(ValueError, "invalid literal for.*XYZ'$", int, 'XYZ') or:: @@ -993,10 +993,10 @@ Test cases | :meth:`assertLessEqual(a, b) | ``a <= b`` | 2.7 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertRegexpMatches(s, re) | ``regex.search(s)`` | 2.7 | + | :meth:`assertRegexpMatches(s, r) | ``r.search(s)`` | 2.7 | | ` | | | +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertNotRegexpMatches(s, re) | ``not regex.search(s)`` | 2.7 | + | :meth:`assertNotRegexpMatches(s, r) | ``not r.search(s)`` | 2.7 | | ` | | | +---------------------------------------+--------------------------------+--------------+ | :meth:`assertItemsEqual(a, b) | sorted(a) == sorted(b) and | 2.7 | @@ -1017,7 +1017,7 @@ Test cases like the :func:`round` function) and not *significant digits*. If *delta* is supplied instead of *places* then the difference - between *first* and *second* must be less (or more) than *delta*. + between *first* and *second* must be less or equal to (or greater than) *delta*. Supplying both *delta* and *places* raises a ``TypeError``. @@ -1075,6 +1075,8 @@ Test cases sorted(actual))`` but it works with sequences of unhashable objects as well. + In Python 3, this method is named ``assertCountEqual``. + .. versionadded:: 2.7 @@ -1601,8 +1603,7 @@ Loading and running tests A list containing 2-tuples of :class:`TestCase` instances and strings holding formatted tracebacks. Each tuple represents a test where a failure - was explicitly signalled using the :meth:`TestCase.fail\*` or - :meth:`TestCase.assert\*` methods. + was explicitly signalled using the :meth:`TestCase.assert\*` methods. .. versionchanged:: 2.2 Contains formatted tracebacks instead of :func:`sys.exc_info` results. @@ -1702,7 +1703,7 @@ Loading and running tests .. method:: addError(test, err) - Called when the test case *test* raises an unexpected exception *err* is a + Called when the test case *test* raises an unexpected exception. *err* is a tuple of the form returned by :func:`sys.exc_info`: ``(type, value, traceback)``. diff --git a/Doc/library/urllib.rst b/Doc/library/urllib.rst index 2008bdfe3e7..c7d200dbe8a 100644 --- a/Doc/library/urllib.rst +++ b/Doc/library/urllib.rst @@ -280,6 +280,13 @@ Utility functions find it, looks for proxy information from Mac OSX System Configuration for Mac OS X and Windows Systems Registry for Windows. +.. note:: + urllib also exposes certain utility functions like splittype, splithost and + others parsing url into various components. But it is recommended to use + :mod:`urlparse` for parsing urls than using these functions directly. + Python 3 does not expose these helper functions from :mod:`urllib.parse` + module. + URL Opener objects ------------------ diff --git a/Doc/library/urllib2.rst b/Doc/library/urllib2.rst index bae5a4b2f5b..a35f1751c2f 100644 --- a/Doc/library/urllib2.rst +++ b/Doc/library/urllib2.rst @@ -60,8 +60,10 @@ The :mod:`urllib2` module defines the following functions: default installed global :class:`OpenerDirector` uses :class:`UnknownHandler` to ensure this never happens). - In addition, default installed :class:`ProxyHandler` makes sure the requests - are handled through the proxy when they are set. + In addition, if proxy settings are detected (for example, when a ``*_proxy`` + environment variable like :envvar:`http_proxy` is set), + :class:`ProxyHandler` is default installed and makes sure the requests are + handled through the proxy. .. versionchanged:: 2.6 *timeout* was added. @@ -83,7 +85,8 @@ The :mod:`urllib2` module defines the following functions: subclasses of :class:`BaseHandler` (in which case it must be possible to call the constructor without any parameters). Instances of the following classes will be in front of the *handler*\s, unless the *handler*\s contain them, - instances of them or subclasses of them: :class:`ProxyHandler`, + instances of them or subclasses of them: :class:`ProxyHandler` (if proxy + settings are detected), :class:`UnknownHandler`, :class:`HTTPHandler`, :class:`HTTPDefaultErrorHandler`, :class:`HTTPRedirectHandler`, :class:`FTPHandler`, :class:`FileHandler`, :class:`HTTPErrorProcessor`. @@ -202,9 +205,9 @@ The following classes are provided: Cause requests to go through a proxy. If *proxies* is given, it must be a dictionary mapping protocol names to URLs of proxies. The default is to read the list of proxies from the environment variables - :envvar:`_proxy`. If no proxy environment variables are set, in a - Windows environment, proxy settings are obtained from the registry's - Internet Settings section and in a Mac OS X environment, proxy information + :envvar:`_proxy`. If no proxy environment variables are set, then + in a Windows environment proxy settings are obtained from the registry's + Internet Settings section, and in a Mac OS X environment proxy information is retrieved from the OS X System Configuration Framework. To disable autodetected proxy pass an empty dictionary. diff --git a/Doc/library/urlparse.rst b/Doc/library/urlparse.rst index efd112d1758..1bc361d8b26 100644 --- a/Doc/library/urlparse.rst +++ b/Doc/library/urlparse.rst @@ -72,7 +72,7 @@ The :mod:`urlparse` module defines the following functions: ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', params='', query='', fragment='') >>> urlparse('www.cwi.nl/%7Eguido/Python.html') - ParseResult(scheme='', netloc='', path='www.cwi.nl:80/%7Eguido/Python.html', + ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', params='', query='', fragment='') >>> urlparse('help/Python.html') ParseResult(scheme='', netloc='', path='help/Python.html', params='', diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 70f0c030761..d4e363ac020 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -58,7 +58,7 @@ instead: .. function:: parseString(string[, parser]) Return a :class:`Document` that represents the *string*. This method creates a - :class:`StringIO` object for the string and passes that on to :func:`parse`. + :class:`~StringIO.StringIO` object for the string and passes that on to :func:`parse`. Both functions return a :class:`Document` object representing the content of the document. diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst index 565dc84907d..541d6496e18 100644 --- a/Doc/library/xml.dom.rst +++ b/Doc/library/xml.dom.rst @@ -441,14 +441,15 @@ objects: In addition, the Python DOM interface requires that some additional support is provided to allow :class:`NodeList` objects to be used as Python sequences. All -:class:`NodeList` implementations must include support for :meth:`__len__` and -:meth:`__getitem__`; this allows iteration over the :class:`NodeList` in +:class:`NodeList` implementations must include support for +:meth:`~object.__len__` and +:meth:`~object.__getitem__`; this allows iteration over the :class:`NodeList` in :keyword:`for` statements and proper support for the :func:`len` built-in function. If a DOM implementation supports modification of the document, the -:class:`NodeList` implementation must also support the :meth:`__setitem__` and -:meth:`__delitem__` methods. +:class:`NodeList` implementation must also support the +:meth:`~object.__setitem__` and :meth:`~object.__delitem__` methods. .. _dom-documenttype-objects: diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index e6ea004b577..24e50846c8f 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -158,7 +158,7 @@ the sub-tree below it (its children, their children, and so on). For example, :meth:`Element.findall` finds only elements with a tag which are direct children of the current element. :meth:`Element.find` finds the *first* child -with a particular tag, and :meth:`Element.text` accesses the element's text +with a particular tag, and :attr:`Element.text` accesses the element's text content. :meth:`Element.get` accesses the element's attributes:: >>> for country in root.findall('country'): @@ -680,8 +680,9 @@ Element Objects or contents. :class:`Element` objects also support the following sequence type methods - for working with subelements: :meth:`__delitem__`, :meth:`__getitem__`, - :meth:`__setitem__`, :meth:`__len__`. + for working with subelements: :meth:`~object.__delitem__`, + :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, + :meth:`~object.__len__`. Caution: Elements with no subelements will test as ``False``. This behavior will change in future versions. Use specific ``len(elem)`` or ``elem is diff --git a/Doc/library/xml.sax.handler.rst b/Doc/library/xml.sax.handler.rst index 23f429e0732..15140a34549 100644 --- a/Doc/library/xml.sax.handler.rst +++ b/Doc/library/xml.sax.handler.rst @@ -240,7 +240,8 @@ events in the input document: Signals the start of an element in non-namespace mode. The *name* parameter contains the raw XML 1.0 name of the element type as a - string and the *attrs* parameter holds an object of the :class:`Attributes` + string and the *attrs* parameter holds an object of the + :class:`~xml.sax.xmlreader.Attributes` interface (see :ref:`attributes-objects`) containing the attributes of the element. The object passed as *attrs* may be re-used by the parser; holding on to a reference to it is not a reliable way to keep a copy of the attributes. @@ -263,7 +264,8 @@ events in the input document: The *name* parameter contains the name of the element type as a ``(uri, localname)`` tuple, the *qname* parameter contains the raw XML 1.0 name used in the source document, and the *attrs* parameter holds an instance of the - :class:`AttributesNS` interface (see :ref:`attributes-ns-objects`) + :class:`~xml.sax.xmlreader.AttributesNS` interface (see + :ref:`attributes-ns-objects`) containing the attributes of the element. If no namespace is associated with the element, the *uri* component of *name* will be ``None``. The object passed as *attrs* may be re-used by the parser; holding on to a reference to it is not @@ -379,8 +381,9 @@ ErrorHandler Objects -------------------- Objects with this interface are used to receive error and warning information -from the :class:`XMLReader`. If you create an object that implements this -interface, then register the object with your :class:`XMLReader`, the parser +from the :class:`~xml.sax.xmlreader.XMLReader`. If you create an object that +implements this interface, then register the object with your +:class:`~xml.sax.xmlreader.XMLReader`, the parser will call the methods in your object to report all warnings and errors. There are three levels of errors available: warnings, (possibly) recoverable errors, and unrecoverable errors. All methods take a :exc:`SAXParseException` as the diff --git a/Doc/library/xml.sax.reader.rst b/Doc/library/xml.sax.reader.rst index 4b3c18ac64c..e09ad7f811d 100644 --- a/Doc/library/xml.sax.reader.rst +++ b/Doc/library/xml.sax.reader.rst @@ -109,47 +109,50 @@ The :class:`XMLReader` interface supports the following methods: .. method:: XMLReader.getContentHandler() - Return the current :class:`ContentHandler`. + Return the current :class:`~xml.sax.handler.ContentHandler`. .. method:: XMLReader.setContentHandler(handler) - Set the current :class:`ContentHandler`. If no :class:`ContentHandler` is set, - content events will be discarded. + Set the current :class:`~xml.sax.handler.ContentHandler`. If no + :class:`~xml.sax.handler.ContentHandler` is set, content events will be + discarded. .. method:: XMLReader.getDTDHandler() - Return the current :class:`DTDHandler`. + Return the current :class:`~xml.sax.handler.DTDHandler`. .. method:: XMLReader.setDTDHandler(handler) - Set the current :class:`DTDHandler`. If no :class:`DTDHandler` is set, DTD + Set the current :class:`~xml.sax.handler.DTDHandler`. If no + :class:`~xml.sax.handler.DTDHandler` is set, DTD events will be discarded. .. method:: XMLReader.getEntityResolver() - Return the current :class:`EntityResolver`. + Return the current :class:`~xml.sax.handler.EntityResolver`. .. method:: XMLReader.setEntityResolver(handler) - Set the current :class:`EntityResolver`. If no :class:`EntityResolver` is set, + Set the current :class:`~xml.sax.handler.EntityResolver`. If no + :class:`~xml.sax.handler.EntityResolver` is set, attempts to resolve an external entity will result in opening the system identifier for the entity, and fail if it is not available. .. method:: XMLReader.getErrorHandler() - Return the current :class:`ErrorHandler`. + Return the current :class:`~xml.sax.handler.ErrorHandler`. .. method:: XMLReader.setErrorHandler(handler) - Set the current error handler. If no :class:`ErrorHandler` is set, errors will - be raised as exceptions, and warnings will be printed. + Set the current error handler. If no :class:`~xml.sax.handler.ErrorHandler` + is set, errors will be raised as exceptions, and warnings will be printed. .. method:: XMLReader.setLocale(locale) @@ -326,8 +329,13 @@ The :class:`Attributes` Interface --------------------------------- :class:`Attributes` objects implement a portion of the mapping protocol, -including the methods :meth:`copy`, :meth:`get`, :meth:`has_key`, :meth:`items`, -:meth:`keys`, and :meth:`values`. The following methods are also provided: +including the methods :meth:`~collections.abc.Mapping.copy`, +:meth:`~collections.abc.Mapping.get`, +:meth:`~collections.abc.Mapping.has_key`, +:meth:`~collections.abc.Mapping.items`, +:meth:`~collections.abc.Mapping.keys`, +and :meth:`~collections.abc.Mapping.values`. The following methods +are also provided: .. method:: Attributes.getLength() diff --git a/Doc/library/xml.sax.rst b/Doc/library/xml.sax.rst index 75cfc119dde..25e4fa90324 100644 --- a/Doc/library/xml.sax.rst +++ b/Doc/library/xml.sax.rst @@ -29,7 +29,8 @@ The convenience functions are: .. function:: make_parser([parser_list]) - Create and return a SAX :class:`XMLReader` object. The first parser found will + Create and return a SAX :class:`~xml.sax.xmlreader.XMLReader` object. The + first parser found will be used. If *parser_list* is provided, it must be a sequence of strings which name modules that have a function named :func:`create_parser`. Modules listed in *parser_list* will be used before modules in the default list of parsers. @@ -39,8 +40,9 @@ The convenience functions are: Create a SAX parser and use it to parse a document. The document, passed in as *filename_or_stream*, can be a filename or a file object. The *handler* - parameter needs to be a SAX :class:`ContentHandler` instance. If - *error_handler* is given, it must be a SAX :class:`ErrorHandler` instance; if + parameter needs to be a SAX :class:`~handler.ContentHandler` instance. If + *error_handler* is given, it must be a SAX :class:`~handler.ErrorHandler` + instance; if omitted, :exc:`SAXParseException` will be raised on all errors. There is no return value; all work must be done by the *handler* passed in. @@ -65,10 +67,12 @@ For these objects, only the interfaces are relevant; they are normally not instantiated by the application itself. Since Python does not have an explicit notion of interface, they are formally introduced as classes, but applications may use implementations which do not inherit from the provided classes. The -:class:`InputSource`, :class:`Locator`, :class:`Attributes`, -:class:`AttributesNS`, and :class:`XMLReader` interfaces are defined in the +:class:`~xml.sax.xmlreader.InputSource`, :class:`~xml.sax.xmlreader.Locator`, +:class:`~xml.sax.xmlreader.Attributes`, :class:`~xml.sax.xmlreader.AttributesNS`, +and :class:`~xml.sax.xmlreader.XMLReader` interfaces are defined in the module :mod:`xml.sax.xmlreader`. The handler interfaces are defined in -:mod:`xml.sax.handler`. For convenience, :class:`InputSource` (which is often +:mod:`xml.sax.handler`. For convenience, +:class:`~xml.sax.xmlreader.InputSource` (which is often instantiated directly) and the handler classes are also available from :mod:`xml.sax`. These interfaces are described below. @@ -81,7 +85,8 @@ classes. Encapsulate an XML error or warning. This class can contain basic error or warning information from either the XML parser or the application: it can be subclassed to provide additional functionality or to add localization. Note - that although the handlers defined in the :class:`ErrorHandler` interface + that although the handlers defined in the + :class:`~xml.sax.handler.ErrorHandler` interface receive instances of this exception, it is not required to actually raise the exception --- it is also useful as a container for information. @@ -94,22 +99,26 @@ classes. .. exception:: SAXParseException(msg, exception, locator) - Subclass of :exc:`SAXException` raised on parse errors. Instances of this class - are passed to the methods of the SAX :class:`ErrorHandler` interface to provide - information about the parse error. This class supports the SAX :class:`Locator` - interface as well as the :class:`SAXException` interface. + Subclass of :exc:`SAXException` raised on parse errors. Instances of this + class are passed to the methods of the SAX + :class:`~xml.sax.handler.ErrorHandler` interface to provide information + about the parse error. This class supports the SAX + :class:`~xml.sax.xmlreader.Locator` interface as well as the + :class:`SAXException` interface. .. exception:: SAXNotRecognizedException(msg[, exception]) - Subclass of :exc:`SAXException` raised when a SAX :class:`XMLReader` is + Subclass of :exc:`SAXException` raised when a SAX + :class:`~xml.sax.xmlreader.XMLReader` is confronted with an unrecognized feature or property. SAX applications and extensions may use this class for similar purposes. .. exception:: SAXNotSupportedException(msg[, exception]) - Subclass of :exc:`SAXException` raised when a SAX :class:`XMLReader` is asked to + Subclass of :exc:`SAXException` raised when a SAX + :class:`~xml.sax.xmlreader.XMLReader` is asked to enable a feature that is not supported, or to set a property to a value that the implementation does not support. SAX applications and extensions may use this class for similar purposes. diff --git a/Doc/library/xml.sax.utils.rst b/Doc/library/xml.sax.utils.rst index c796ca8c86e..e774b6b8cb2 100644 --- a/Doc/library/xml.sax.utils.rst +++ b/Doc/library/xml.sax.utils.rst @@ -59,7 +59,8 @@ or as base classes. .. class:: XMLGenerator([out[, encoding]]) - This class implements the :class:`ContentHandler` interface by writing SAX + This class implements the :class:`~xml.sax.handler.ContentHandler` interface + by writing SAX events back into an XML document. In other words, using an :class:`XMLGenerator` as the content handler will reproduce the original document being parsed. *out* should be a file-like object which will default to *sys.stdout*. *encoding* is @@ -68,7 +69,8 @@ or as base classes. .. class:: XMLFilterBase(base) - This class is designed to sit between an :class:`XMLReader` and the client + This class is designed to sit between an + :class:`~xml.sax.xmlreader.XMLReader` and the client application's event handlers. By default, it does nothing but pass requests up to the reader and events on to the handlers unmodified, but subclasses can override specific methods to modify the event stream or the configuration @@ -77,9 +79,10 @@ or as base classes. .. function:: prepare_input_source(source[, base]) - This function takes an input source and an optional base URL and returns a fully - resolved :class:`InputSource` object ready for reading. The input source can be - given as a string, a file-like object, or an :class:`InputSource` object; - parsers will use this function to implement the polymorphic *source* argument to - their :meth:`parse` method. + This function takes an input source and an optional base URL and returns a + fully resolved :class:`~xml.sax.xmlreader.InputSource` object ready for + reading. The input source can be given as a string, a file-like object, or + an :class:`~xml.sax.xmlreader.InputSource` object; parsers will use this + function to implement the polymorphic *source* argument to their + :meth:`parse` method. diff --git a/Doc/license.rst b/Doc/license.rst index 7b519a9b990..2f8546eb8c4 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -50,59 +50,11 @@ been GPL-compatible; the table below summarizes the various releases. +----------------+--------------+-----------+------------+-----------------+ | 2.1.1 | 2.1+2.0.1 | 2001 | PSF | yes | +----------------+--------------+-----------+------------+-----------------+ -| 2.2 | 2.1.1 | 2001 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ | 2.1.2 | 2.1.1 | 2002 | PSF | yes | +----------------+--------------+-----------+------------+-----------------+ | 2.1.3 | 2.1.2 | 2002 | PSF | yes | +----------------+--------------+-----------+------------+-----------------+ -| 2.2.1 | 2.2 | 2002 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.2.2 | 2.2.1 | 2002 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.2.3 | 2.2.2 | 2002-2003 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.3 | 2.2.2 | 2002-2003 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.3.1 | 2.3 | 2002-2003 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.3.2 | 2.3.1 | 2003 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.3.3 | 2.3.2 | 2003 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.3.4 | 2.3.3 | 2004 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.3.5 | 2.3.4 | 2005 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.4 | 2.3 | 2004 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.4.1 | 2.4 | 2005 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.4.2 | 2.4.1 | 2005 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.4.3 | 2.4.2 | 2006 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.4.4 | 2.4.3 | 2006 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.5 | 2.4 | 2006 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.5.1 | 2.5 | 2007 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.5.2 | 2.5.1 | 2008 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.5.3 | 2.5.2 | 2008 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.6 | 2.5 | 2008 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.6.1 | 2.6 | 2008 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.6.2 | 2.6.1 | 2009 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.6.3 | 2.6.2 | 2009 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.6.4 | 2.6.3 | 2010 | PSF | yes | -+----------------+--------------+-----------+------------+-----------------+ -| 2.7 | 2.6 | 2010 | PSF | yes | +| 2.2 and above | 2.1.1 | 2001-now | PSF | yes | +----------------+--------------+-----------+------------+-----------------+ .. note:: diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 7b6c3e22246..4a9ad08200a 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -509,14 +509,14 @@ receiving any excess positional parameters, defaulting to the empty tuple. If the form "``**identifier``" is present, it is initialized to a new dictionary receiving any excess keyword arguments, defaulting to a new empty dictionary. -.. index:: pair: lambda; form +.. index:: pair: lambda; expression It is also possible to create anonymous functions (functions not bound to a -name), for immediate use in expressions. This uses lambda forms, described in -section :ref:`lambda`. Note that the lambda form is merely a shorthand for a +name), for immediate use in expressions. This uses lambda expressions, described in +section :ref:`lambda`. Note that the lambda expression is merely a shorthand for a simplified function definition; a function defined in a ":keyword:`def`" statement can be passed around or assigned to another name just like a function -defined by a lambda form. The ":keyword:`def`" form is actually more powerful +defined by a lambda expression. The ":keyword:`def`" form is actually more powerful since it allows the execution of multiple statements. **Programmer's note:** Functions are first-class objects. A "``def``" form diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 79114c1ec25..b57b65e7acb 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -411,7 +411,7 @@ Set types These represent a mutable set. They are created by the built-in :func:`set` constructor and can be modified afterwards by several methods, such as - :meth:`add`. + :meth:`~set.add`. Frozen sets .. index:: object: frozenset @@ -623,9 +623,8 @@ Callable types single: im_self (method attribute) When a user-defined method object is created by retrieving a class method object - from a class or instance, its :attr:`im_self` attribute is the class itself (the - same as the :attr:`im_class` attribute), and its :attr:`im_func` attribute is - the function object underlying the class method. + from a class or instance, its :attr:`im_self` attribute is the class itself, and + its :attr:`im_func` attribute is the function object underlying the class method. When an unbound user-defined method object is called, the underlying function (:attr:`im_func`) is called, with the restriction that the first argument must @@ -662,7 +661,8 @@ Callable types :ref:`yield`) is called a :dfn:`generator function`. Such a function, when called, always returns an iterator object which can be used to execute the body of the function: calling the iterator's - :meth:`next` method will cause the function to execute until it provides a value + :meth:`~iterator.next` method will cause the function to execute until + it provides a value using the :keyword:`yield` statement. When the function executes a :keyword:`return` statement or falls off the end, a :exc:`StopIteration` exception is raised and the iterator will have reached the end of the set of @@ -797,8 +797,8 @@ Classes associated class is either :class:`C` or one of its base classes, it is transformed into an unbound user-defined method object whose :attr:`im_class` attribute is :class:`C`. When it would yield a class method object, it is - transformed into a bound user-defined method object whose :attr:`im_class` - and :attr:`im_self` attributes are both :class:`C`. When it would yield a + transformed into a bound user-defined method object whose + :attr:`im_self` attribute is :class:`C`. When it would yield a static method object, it is transformed into the object wrapped by the static method object. See section :ref:`descriptors` for another way in which attributes retrieved from a class may differ from those actually contained in @@ -822,10 +822,10 @@ Classes Special attributes: :attr:`__name__` is the class name; :attr:`__module__` is the module name in which the class was defined; :attr:`__dict__` is the - dictionary containing the class's namespace; :attr:`__bases__` is a tuple - (possibly empty or a singleton) containing the base classes, in the order of - their occurrence in the base class list; :attr:`__doc__` is the class's - documentation string, or None if undefined. + dictionary containing the class's namespace; :attr:`~class.__bases__` is a + tuple (possibly empty or a singleton) containing the base classes, in the + order of their occurrence in the base class list; :attr:`__doc__` is the + class's documentation string, or None if undefined. Class instances .. index:: @@ -870,8 +870,8 @@ Class instances single: __dict__ (instance attribute) single: __class__ (instance attribute) - Special attributes: :attr:`__dict__` is the attribute dictionary; - :attr:`__class__` is the instance's class. + Special attributes: :attr:`~object.__dict__` is the attribute dictionary; + :attr:`~instance.__class__` is the instance's class. Files .. index:: @@ -1070,9 +1070,9 @@ Internal types single: stop (slice object attribute) single: step (slice object attribute) - Special read-only attributes: :attr:`start` is the lower bound; :attr:`stop` is - the upper bound; :attr:`step` is the step value; each is ``None`` if omitted. - These attributes can have any type. + Special read-only attributes: :attr:`~slice.start` is the lower bound; + :attr:`~slice.stop` is the upper bound; :attr:`~slice.step` is the step + value; each is ``None`` if omitted. These attributes can have any type. Slice objects support one method: @@ -1179,7 +1179,8 @@ When implementing a class that emulates any built-in type, it is important that the emulation only be implemented to the degree that it makes sense for the object being modelled. For example, some sequences may work well with retrieval of individual elements, but extracting a slice may not make sense. (One example -of this is the :class:`NodeList` interface in the W3C's Document Object Model.) +of this is the :class:`~xml.dom.NodeList` interface in the W3C's Document +Object Model.) .. _customization: @@ -1811,10 +1812,10 @@ case the instance is itself a class. :pep:`3119` - Introducing Abstract Base Classes Includes the specification for customizing :func:`isinstance` and - :func:`issubclass` behavior through :meth:`__instancecheck__` and - :meth:`__subclasscheck__`, with motivation for this functionality in the - context of adding Abstract Base Classes (see the :mod:`abc` module) to the - language. + :func:`issubclass` behavior through :meth:`~class.__instancecheck__` and + :meth:`~class.__subclasscheck__`, with motivation for this functionality + in the context of adding Abstract Base Classes (see the :mod:`abc` + module) to the language. .. _callable-types: @@ -1847,7 +1848,7 @@ range of items. (For backwards compatibility, the method :meth:`__getslice__` is also recommended that mappings provide the methods :meth:`keys`, :meth:`values`, :meth:`items`, :meth:`has_key`, :meth:`get`, :meth:`clear`, :meth:`setdefault`, :meth:`iterkeys`, :meth:`itervalues`, :meth:`iteritems`, -:meth:`pop`, :meth:`popitem`, :meth:`copy`, and :meth:`update` behaving similar +:meth:`pop`, :meth:`popitem`, :meth:`!copy`, and :meth:`update` behaving similar to those for Python's standard dictionary objects. The :mod:`UserDict` module provides a :class:`DictMixin` class to help create those methods from a base set of :meth:`__getitem__`, :meth:`__setitem__`, :meth:`__delitem__`, and diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index a11ee6e2624..d3dc1112b5e 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -96,14 +96,13 @@ exception. definition begins with two or more underscore characters and does not end in two or more underscores, it is considered a :dfn:`private name` of that class. Private names are transformed to a longer form before code is generated for -them. The transformation inserts the class name in front of the name, with -leading underscores removed, and a single underscore inserted in front of the -class name. For example, the identifier ``__spam`` occurring in a class named -``Ham`` will be transformed to ``_Ham__spam``. This transformation is -independent of the syntactical context in which the identifier is used. If the -transformed name is extremely long (longer than 255 characters), implementation -defined truncation may happen. If the class name consists only of underscores, -no transformation is done. +them. The transformation inserts the class name, with leading underscores +removed and a single underscore inserted, in front of the name. For example, +the identifier ``__spam`` occurring in a class named ``Ham`` will be transformed +to ``_Ham__spam``. This transformation is independent of the syntactical +context in which the identifier is used. If the transformed name is extremely +long (longer than 255 characters), implementation defined truncation may happen. +If the class name consists only of underscores, no transformation is done. @@ -185,7 +184,7 @@ brackets: list_comprehension: `expression` `list_for` list_for: "for" `target_list` "in" `old_expression_list` [`list_iter`] old_expression_list: `old_expression` [("," `old_expression`)+ [","]] - old_expression: `or_test` | `old_lambda_form` + old_expression: `or_test` | `old_lambda_expr` list_iter: `list_for` | `list_if` list_if: "if" `old_expression` [`list_iter`] @@ -432,6 +431,7 @@ Note that calling any of the generator methods below when the generator is already executing raises a :exc:`ValueError` exception. .. index:: exception: StopIteration +.. class:: generator .. method:: generator.next() @@ -445,6 +445,7 @@ is already executing raises a :exc:`ValueError` exception. exits without yielding another value, a :exc:`StopIteration` exception is raised. +.. class:: . .. method:: generator.send(value) @@ -661,10 +662,10 @@ is a tuple containing the conversion of the slice items; otherwise, the conversion of the lone slice item is the key. The conversion of a slice item that is an expression is that expression. The conversion of an ellipsis slice item is the built-in ``Ellipsis`` object. The conversion of a proper slice is a -slice object (see section :ref:`types`) whose :attr:`start`, :attr:`stop` and -:attr:`step` attributes are the values of the expressions given as lower bound, -upper bound and stride, respectively, substituting ``None`` for missing -expressions. +slice object (see section :ref:`types`) whose :attr:`~slice.start`, +:attr:`~slice.stop` and :attr:`~slice.step` attributes are the values of the +expressions given as lower bound, upper bound and stride, respectively, +substituting ``None`` for missing expressions. .. index:: @@ -1256,7 +1257,7 @@ Conditional Expressions .. productionlist:: conditional_expression: `or_test` ["if" `or_test` "else" `expression`] - expression: `conditional_expression` | `lambda_form` + expression: `conditional_expression` | `lambda_expr` Conditional expressions (sometimes called a "ternary operator") have the lowest priority of all Python operations. @@ -1276,14 +1277,13 @@ Lambdas .. index:: pair: lambda; expression - pair: lambda; form pair: anonymous; function .. productionlist:: - lambda_form: "lambda" [`parameter_list`]: `expression` - old_lambda_form: "lambda" [`parameter_list`]: `old_expression` + lambda_expr: "lambda" [`parameter_list`]: `expression` + old_lambda_expr: "lambda" [`parameter_list`]: `old_expression` -Lambda forms (lambda expressions) have the same syntactic position as +Lambda expressions (sometimes called lambda forms) have the same syntactic position as expressions. They are a shorthand to create anonymous functions; the expression ``lambda arguments: expression`` yields a function object. The unnamed object behaves like a function object defined with :: @@ -1292,7 +1292,7 @@ behaves like a function object defined with :: return expression See section :ref:`function` for the syntax of parameter lists. Note that -functions created with lambda forms cannot contain statements. +functions created with lambda expressions cannot contain statements. .. _exprlists: diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index ea92b8ed87a..2f31c853f50 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -529,8 +529,7 @@ Notes: (2) Any Unicode character can be encoded this way, but characters outside the Basic Multilingual Plane (BMP) will be encoded using a surrogate pair if Python is - compiled to use 16-bit code units (the default). Individual code units which - form parts of a surrogate pair can be encoded using this escape sequence. + compiled to use 16-bit code units (the default). (3) As in Standard C, up to three octal digits are accepted. diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index d8cfd5c6ca5..254d1ba1147 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -511,6 +511,9 @@ reference count or by being garbage collected), the generator-iterator's :meth:`close` method will be called, allowing any pending :keyword:`finally` clauses to execute. +For full details of :keyword:`yield` semantics, refer to the :ref:`yieldexpr` +section. + .. note:: In Python 2.2, the :keyword:`yield` statement was only allowed when the @@ -978,15 +981,16 @@ The :keyword:`exec` statement exec_stmt: "exec" `or_expr` ["in" `expression` ["," `expression`]] This statement supports dynamic execution of Python code. The first expression -should evaluate to either a string, an open file object, a code object, or a -tuple. If it is a string, the string is parsed as a suite of Python statements -which is then executed (unless a syntax error occurs). [#]_ If it is an open -file, the file is parsed until EOF and executed. If it is a code object, it is -simply executed. For the interpretation of a tuple, see below. In all cases, -the code that's executed is expected to be valid as file input (see section -:ref:`file-input`). Be aware that the :keyword:`return` and :keyword:`yield` -statements may not be used outside of function definitions even within the -context of code passed to the :keyword:`exec` statement. +should evaluate to either a Unicode string, a *Latin-1* encoded string, an open +file object, a code object, or a tuple. If it is a string, the string is parsed +as a suite of Python statements which is then executed (unless a syntax error +occurs). [#]_ If it is an open file, the file is parsed until EOF and executed. +If it is a code object, it is simply executed. For the interpretation of a +tuple, see below. In all cases, the code that's executed is expected to be +valid as file input (see section :ref:`file-input`). Be aware that the +:keyword:`return` and :keyword:`yield` statements may not be used outside of +function definitions even within the context of code passed to the +:keyword:`exec` statement. In all cases, if the optional parts are omitted, the code is executed in the current scope. If only the first expression after ``in`` is specified, diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 2cc3610c220..eadeeacf321 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -609,7 +609,7 @@ will do nicely:: A piece of Python code that expects a particular abstract data type can often be passed a class that emulates the methods of that data type instead. For instance, if you have a function that formats some data from a file object, you -can define a class with methods :meth:`read` and :meth:`readline` that get the +can define a class with methods :meth:`read` and :meth:`!readline` that get the data from a string buffer instead, and pass it as an argument. .. (Unfortunately, this technique has its limitations: a class can't define @@ -693,9 +693,10 @@ using a :keyword:`for` statement:: This style of access is clear, concise, and convenient. The use of iterators pervades and unifies Python. Behind the scenes, the :keyword:`for` statement calls :func:`iter` on the container object. The function returns an iterator -object that defines the method :meth:`next` which accesses elements in the -container one at a time. When there are no more elements, :meth:`next` raises a -:exc:`StopIteration` exception which tells the :keyword:`for` loop to terminate. +object that defines the method :meth:`~iterator.next` which accesses elements +in the container one at a time. When there are no more elements, +:meth:`~iterator.next` raises a :exc:`StopIteration` exception which tells the +:keyword:`for` loop to terminate. This example shows how it all works:: >>> s = 'abc' diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index aad26720119..4de9ec927f3 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -541,17 +541,16 @@ In the same fashion, dictionaries can deliver keyword arguments with the ``**``\ .. _tut-lambda: -Lambda Forms ------------- - -By popular demand, a few features commonly found in functional programming -languages like Lisp have been added to Python. With the :keyword:`lambda` -keyword, small anonymous functions can be created. Here's a function that -returns the sum of its two arguments: ``lambda a, b: a+b``. Lambda forms can be -used wherever function objects are required. They are syntactically restricted -to a single expression. Semantically, they are just syntactic sugar for a -normal function definition. Like nested function definitions, lambda forms can -reference variables from the containing scope:: +Lambda Expressions +------------------ + +Small anonymous functions can be created with the :keyword:`lambda` keyword. +This function returns the sum of its two arguments: ``lambda a, b: a+b``. +Lambda functions can be used wherever function objects are required. They are +syntactically restricted to a single expression. Semantically, they are just +syntactic sugar for a normal function definition. Like nested function +definitions, lambda functions can reference variables from the containing +scope:: >>> def make_incrementor(n): ... return lambda x: x + n @@ -562,6 +561,14 @@ reference variables from the containing scope:: >>> f(1) 43 +The above example uses a lambda expression to return a function. Another use +is to pass a small function as an argument:: + + >>> pairs = [(1, 'one'), (2, 'two'), (3, 'three'), (4, 'four')] + >>> pairs.sort(key=lambda pair: pair[1]) + >>> pairs + [(4, 'four'), (1, 'one'), (3, 'three'), (2, 'two')] + .. _tut-docstrings: diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index e6786cc0e27..28e6ad71b95 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -171,7 +171,7 @@ There are three built-in functions that are very useful when used with lists: the sequence for which ``function(item)`` is true. If *sequence* is a :class:`string` or :class:`tuple`, the result will be of the same type; otherwise, it is always a :class:`list`. For example, to compute a sequence of -numbers not divisible by 2 and 3:: +numbers not divisible by 2 or 3:: >>> def f(x): return x % 2 != 0 and x % 3 != 0 ... diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 2f08110feef..45ca79244c4 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -215,10 +215,6 @@ operation. For example:: >>> print 'The value of PI is approximately %5.3f.' % math.pi The value of PI is approximately 3.142. -Since :meth:`str.format` is quite new, a lot of Python code still uses the ``%`` -operator. However, because this old style of formatting will eventually be -removed from the language, :meth:`str.format` should generally be used. - More information can be found in the :ref:`string-formatting` section. @@ -295,18 +291,8 @@ containing only a single newline. :: >>> f.readline() '' -``f.readlines()`` returns a list containing all the lines of data in the file. -If given an optional parameter *sizehint*, it reads that many bytes from the -file and enough more to complete a line, and returns the lines from that. This -is often used to allow efficient reading of a large file by lines, but without -having to load the entire file in memory. Only complete lines will be returned. -:: - - >>> f.readlines() - ['This is the first line of the file.\n', 'Second line of the file\n'] - -An alternative approach to reading lines is to loop over the file object. This is -memory efficient, fast, and leads to simpler code:: +For reading lines from a file, you can loop over the file object. This is memory +efficient, fast, and leads to simple code:: >>> for line in f: print line, @@ -314,9 +300,8 @@ memory efficient, fast, and leads to simpler code:: This is the first line of the file. Second line of the file -The alternative approach is simpler but does not provide as fine-grained -control. Since the two approaches manage line buffering differently, they -should not be mixed. +If you want to read all the lines of a file in a list you can also use +``list(f)`` or ``f.readlines()``. ``f.write(string)`` writes the contents of *string* to the file, returning ``None``. :: diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index 6bdc1c5ba4e..5140b433848 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -185,8 +185,8 @@ encodings can be found in the Python Library Reference, in the section on For example, to write Unicode literals including the Euro currency symbol, the ISO-8859-15 encoding can be used, with the Euro symbol having the ordinal value -164. This script will print the value 8364 (the Unicode codepoint corresponding -to the Euro symbol) and then exit:: +164. This script, when saved in the ISO-8859-15 encoding, will print the value +8364 (the Unicode codepoint corresponding to the Euro symbol) and then exit:: # -*- coding: iso-8859-15 -*- @@ -252,7 +252,7 @@ of your user site-packages directory. Start Python and run this code: >>> import site >>> site.getusersitepackages() - '/home/user/.local/lib/python3.2/site-packages' + '/home/user/.local/lib/python2.7/site-packages' Now you can create a file named :file:`usercustomize.py` in that directory and put anything you want in it. It will affect every invocation of Python, unless diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index d1a72c082bf..f5b811452ca 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -71,7 +71,8 @@ More on Modules A module can contain executable statements as well as function definitions. These statements are intended to initialize the module. They are executed only -the *first* time the module is imported somewhere. [#]_ +the *first* time the module name is encountered in an import statement. [#]_ +(They are also run if the file is executed as a script.) Each module has its own private symbol table, which is used as the global symbol table by all functions defined in the module. Thus, the author of a module can @@ -367,7 +368,9 @@ There are also many different operations you might want to perform on sound data (such as mixing, adding echo, applying an equalizer function, creating an artificial stereo effect), so in addition you will be writing a never-ending stream of modules to perform these operations. Here's a possible structure for -your package (expressed in terms of a hierarchical filesystem):: +your package (expressed in terms of a hierarchical filesystem): + +.. code-block:: text sound/ Top-level package __init__.py Initialize the sound package @@ -464,7 +467,7 @@ list of module names that should be imported when ``from package import *`` is encountered. It is up to the package author to keep this list up-to-date when a new version of the package is released. Package authors may also decide not to support it, if they don't see a use for importing \* from their package. For -example, the file :file:`sounds/effects/__init__.py` could contain the following +example, the file :file:`sound/effects/__init__.py` could contain the following code:: __all__ = ["echo", "surround", "reverse"] @@ -550,6 +553,6 @@ modules found in a package. .. rubric:: Footnotes .. [#] In fact function definitions are also 'statements' that are 'executed'; the - execution of a module-level function enters the function name in the module's - global symbol table. + execution of a module-level function definition enters the function name in + the module's global symbol table. diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index 1b0e856a67d..65fb093a1db 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -71,9 +71,9 @@ formatting numbers with group separators:: Templating ========== -The :mod:`string` module includes a versatile :class:`Template` class with a -simplified syntax suitable for editing by end-users. This allows users to -customize their applications without having to alter the application. +The :mod:`string` module includes a versatile :class:`~string.Template` class +with a simplified syntax suitable for editing by end-users. This allows users +to customize their applications without having to alter the application. The format uses placeholder names formed by ``$`` with valid Python identifiers (alphanumeric characters and underscores). Surrounding the placeholder with @@ -85,11 +85,11 @@ spaces. Writing ``$$`` creates a single escaped ``$``:: >>> t.substitute(village='Nottingham', cause='the ditch fund') 'Nottinghamfolk send $10 to the ditch fund.' -The :meth:`substitute` method raises a :exc:`KeyError` when a placeholder is not -supplied in a dictionary or a keyword argument. For mail-merge style -applications, user supplied data may be incomplete and the -:meth:`safe_substitute` method may be more appropriate --- it will leave -placeholders unchanged if data is missing:: +The :meth:`~string.Template.substitute` method raises a :exc:`KeyError` when a +placeholder is not supplied in a dictionary or a keyword argument. For +mail-merge style applications, user supplied data may be incomplete and the +:meth:`~string.Template.safe_substitute` method may be more appropriate --- +it will leave placeholders unchanged if data is missing:: >>> t = Template('Return the $item to $owner.') >>> d = dict(item='unladen swallow') @@ -132,8 +132,9 @@ templates for XML files, plain text reports, and HTML web reports. Working with Binary Data Record Layouts ======================================= -The :mod:`struct` module provides :func:`pack` and :func:`unpack` functions for -working with variable length binary record formats. The following example shows +The :mod:`struct` module provides :func:`~struct.pack` and +:func:`~struct.unpack` functions for working with variable length binary +record formats. The following example shows how to loop through header information in a ZIP file without using the :mod:`zipfile` module. Pack codes ``"H"`` and ``"I"`` represent two and four byte unsigned numbers respectively. The ``"<"`` indicates that they are @@ -229,8 +230,9 @@ This produces the following output: By default, informational and debugging messages are suppressed and the output is sent to standard error. Other output options include routing messages through email, datagrams, sockets, or to an HTTP Server. New filters can select -different routing based on message priority: :const:`DEBUG`, :const:`INFO`, -:const:`WARNING`, :const:`ERROR`, and :const:`CRITICAL`. +different routing based on message priority: :const:`~logging.DEBUG`, +:const:`~logging.INFO`, :const:`~logging.WARNING`, :const:`~logging.ERROR`, +and :const:`~logging.CRITICAL`. The logging system can be configured directly from Python or can be loaded from a user editable configuration file for customized logging without altering the @@ -287,11 +289,11 @@ Many data structure needs can be met with the built-in list type. However, sometimes there is a need for alternative implementations with different performance trade-offs. -The :mod:`array` module provides an :class:`array()` object that is like a list -that stores only homogeneous data and stores it more compactly. The following -example shows an array of numbers stored as two byte unsigned binary numbers -(typecode ``"H"``) rather than the usual 16 bytes per entry for regular lists of -Python int objects:: +The :mod:`array` module provides an :class:`~array.array()` object that is like +a list that stores only homogeneous data and stores it more compactly. The +following example shows an array of numbers stored as two byte unsigned binary +numbers (typecode ``"H"``) rather than the usual 16 bytes per entry for regular +lists of Python int objects:: >>> from array import array >>> a = array('H', [4000, 10, 700, 22222]) @@ -300,10 +302,10 @@ Python int objects:: >>> a[1:3] array('H', [10, 700]) -The :mod:`collections` module provides a :class:`deque()` object that is like a -list with faster appends and pops from the left side but slower lookups in the -middle. These objects are well suited for implementing queues and breadth first -tree searches:: +The :mod:`collections` module provides a :class:`~collections.deque()` object +that is like a list with faster appends and pops from the left side but slower +lookups in the middle. These objects are well suited for implementing queues +and breadth first tree searches:: >>> from collections import deque >>> d = deque(["task1", "task2", "task3"]) @@ -349,8 +351,8 @@ not want to run a full list sort:: Decimal Floating Point Arithmetic ================================= -The :mod:`decimal` module offers a :class:`Decimal` datatype for decimal -floating point arithmetic. Compared to the built-in :class:`float` +The :mod:`decimal` module offers a :class:`~decimal.Decimal` datatype for +decimal floating point arithmetic. Compared to the built-in :class:`float` implementation of binary floating point, the class is especially helpful for * financial applications and other uses which require exact decimal @@ -374,13 +376,15 @@ becomes significant if the results are rounded to the nearest cent:: >>> round(.70 * 1.05, 2) # same calculation with floats 0.73 -The :class:`Decimal` result keeps a trailing zero, automatically inferring four -place significance from multiplicands with two place significance. Decimal -reproduces mathematics as done by hand and avoids issues that can arise when -binary floating point cannot exactly represent decimal quantities. +The :class:`~decimal.Decimal` result keeps a trailing zero, automatically +inferring four place significance from multiplicands with two place +significance. Decimal reproduces mathematics as done by hand and avoids +issues that can arise when binary floating point cannot exactly represent +decimal quantities. -Exact representation enables the :class:`Decimal` class to perform modulo -calculations and equality tests that are unsuitable for binary floating point:: +Exact representation enables the :class:`~decimal.Decimal` class to perform +modulo calculations and equality tests that are unsuitable for binary floating +point:: >>> Decimal('1.00') % Decimal('.10') Decimal('0.00') diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 7dcc1b23f9c..5f29812e448 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -25,14 +25,14 @@ installers for the latest 2.3 release for Mac OS 9 and related documentation. Getting and Installing MacPython ================================ -Mac OS X 10.5 comes with Python 2.5.1 pre-installed by Apple. If you wish, you +Mac OS X 10.8 comes with Python 2.7 pre-installed by Apple. If you wish, you are invited to install the most recent version of Python from the Python website (http://www.python.org). A current "universal binary" build of Python, which runs natively on the Mac's new Intel and legacy PPC CPU's, is available there. What you get after installing is a number of things: -* A :file:`MacPython 2.5` folder in your :file:`Applications` folder. In here +* A :file:`MacPython 2.7` folder in your :file:`Applications` folder. In here you find IDLE, the development environment that is a standard part of official Python distributions; PythonLauncher, which handles double-clicking Python scripts from the Finder; and the "Build Applet" tool, which allows you to @@ -100,7 +100,7 @@ aware of: programs that talk to the Aqua window manager (in other words, anything that has a GUI) need to be run in a special way. Use :program:`pythonw` instead of :program:`python` to start such scripts. -With Python 2.5, you can use either :program:`python` or :program:`pythonw`. +With Python 2.7, you can use either :program:`python` or :program:`pythonw`. Configuration @@ -133,13 +133,11 @@ Installing Additional Python Packages There are several methods to install additional Python packages: -* http://pythonmac.org/packages/ contains selected compiled packages for Python - 2.5, 2.4, and 2.3. - * Packages can be installed via the standard Python distutils mode (``python setup.py install``). -* Many packages can also be installed via the :program:`setuptools` extension. +* Many packages can also be installed via the :program:`setuptools` extension + or :program:`pip` wrapper, see http://www.pip-installer.org/. GUI Programming on the Mac @@ -167,7 +165,7 @@ http://www.riverbankcomputing.co.uk/software/pyqt/intro. Distributing Python Applications on the Mac =========================================== -The "Build Applet" tool that is placed in the MacPython 2.5 folder is fine for +The "Build Applet" tool that is placed in the MacPython 2.7 folder is fine for packaging small Python scripts on your own machine to run as a standard Mac application. This tool, however, is not robust enough to distribute Python applications to other users. diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index cb690a6b630..ec5ac6c65d5 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -28,7 +28,7 @@ following links: http://www.debian.org/doc/manuals/maint-guide/first.en.html for Debian users - http://linuxmafia.com/pub/linux/suse-linux-internals/chapter35.html + http://en.opensuse.org/Portal:Packaging for OpenSuse users http://docs.fedoraproject.org/en-US/Fedora_Draft_Documentation/0.1/html/RPM_Guide/ch-creating-rpms.html for Fedora users diff --git a/Doc/whatsnew/2.2.rst b/Doc/whatsnew/2.2.rst index 412c1d0435b..5f6fe11d778 100644 --- a/Doc/whatsnew/2.2.rst +++ b/Doc/whatsnew/2.2.rst @@ -450,9 +450,9 @@ signal that the iterator is done. Python classes can define an :meth:`__iter__` method, which should create and return a new iterator for the object; if the object is its own iterator, this method can just return ``self``. In particular, iterators will usually be their -own iterators. Extension types implemented in C can implement a :attr:`tp_iter` +own iterators. Extension types implemented in C can implement a :c:member:`~PyTypeObject.tp_iter` function in order to return an iterator, and extension types that want to behave -as iterators can define a :attr:`tp_iternext` function. +as iterators can define a :c:member:`~PyTypeObject.tp_iternext` function. So, after all this, what do iterators actually do? They have one required method, :meth:`next`, which takes no arguments and returns the next value. When @@ -478,7 +478,7 @@ there are no more values to be returned, calling :meth:`next` should raise the In 2.2, Python's :keyword:`for` statement no longer expects a sequence; it expects something for which :func:`iter` will return an iterator. For backward compatibility and convenience, an iterator is automatically constructed for -sequences that don't implement :meth:`__iter__` or a :attr:`tp_iter` slot, so +sequences that don't implement :meth:`__iter__` or a :c:member:`~PyTypeObject.tp_iter` slot, so ``for i in [1,2,3]`` will still work. Wherever the Python interpreter loops over a sequence, it's been changed to use the iterator protocol. This means you can do things like this:: diff --git a/Include/datetime.h b/Include/datetime.h index 47abe5cb865..c0e7ffd00a9 100644 --- a/Include/datetime.h +++ b/Include/datetime.h @@ -42,7 +42,7 @@ typedef struct typedef struct { - PyObject_HEAD /* a pure abstract base clase */ + PyObject_HEAD /* a pure abstract base class */ } PyDateTime_TZInfo; diff --git a/Include/object.h b/Include/object.h index 1ba33eb6c1c..afbc68dc04e 100644 --- a/Include/object.h +++ b/Include/object.h @@ -984,16 +984,22 @@ PyAPI_FUNC(void) _PyTrash_thread_destroy_chain(void); #define PyTrash_UNWIND_LEVEL 50 +/* Note the workaround for when the thread state is NULL (issue #17703) */ #define Py_TRASHCAN_SAFE_BEGIN(op) \ do { \ PyThreadState *_tstate = PyThreadState_GET(); \ - if (_tstate->trash_delete_nesting < PyTrash_UNWIND_LEVEL) { \ - ++_tstate->trash_delete_nesting; + if (!_tstate || \ + _tstate->trash_delete_nesting < PyTrash_UNWIND_LEVEL) { \ + if (_tstate) \ + ++_tstate->trash_delete_nesting; /* The body of the deallocator is here. */ #define Py_TRASHCAN_SAFE_END(op) \ - --_tstate->trash_delete_nesting; \ - if (_tstate->trash_delete_later && _tstate->trash_delete_nesting <= 0) \ - _PyTrash_thread_destroy_chain(); \ + if (_tstate) { \ + --_tstate->trash_delete_nesting; \ + if (_tstate->trash_delete_later \ + && _tstate->trash_delete_nesting <= 0) \ + _PyTrash_thread_destroy_chain(); \ + } \ } \ else \ _PyTrash_thread_deposit_object((PyObject*)op); \ diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 7d6952e1d09..07b79e41fb5 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -22,12 +22,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 2 #define PY_MINOR_VERSION 7 -#define PY_MICRO_VERSION 4 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA -#define PY_RELEASE_SERIAL 1 +#define PY_MICRO_VERSION 5 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "2.7.4rc1+" +#define PY_VERSION "2.7.5+" /*--end constants--*/ /* Subversion Revision number of this file (not of the repository). Empty diff --git a/Include/weakrefobject.h b/Include/weakrefobject.h index e46aecf3858..e7c0eae5398 100644 --- a/Include/weakrefobject.h +++ b/Include/weakrefobject.h @@ -49,9 +49,6 @@ PyAPI_DATA(PyTypeObject) _PyWeakref_CallableProxyType; ((Py_TYPE(op) == &_PyWeakref_ProxyType) || \ (Py_TYPE(op) == &_PyWeakref_CallableProxyType)) -/* This macro calls PyWeakref_CheckRef() last since that can involve a - function call; this makes it more likely that the function call - will be avoided. */ #define PyWeakref_Check(op) \ (PyWeakref_CheckRef(op) || PyWeakref_CheckProxy(op)) diff --git a/LICENSE b/LICENSE index 2e89c09a244..56a5d5cc7f1 100644 --- a/LICENSE +++ b/LICENSE @@ -36,34 +36,9 @@ the various releases. 2.1 2.0+1.6.1 2001 PSF no 2.0.1 2.0+1.6.1 2001 PSF yes 2.1.1 2.1+2.0.1 2001 PSF yes - 2.2 2.1.1 2001 PSF yes 2.1.2 2.1.1 2002 PSF yes 2.1.3 2.1.2 2002 PSF yes - 2.2.1 2.2 2002 PSF yes - 2.2.2 2.2.1 2002 PSF yes - 2.2.3 2.2.2 2003 PSF yes - 2.3 2.2.2 2002-2003 PSF yes - 2.3.1 2.3 2002-2003 PSF yes - 2.3.2 2.3.1 2002-2003 PSF yes - 2.3.3 2.3.2 2002-2003 PSF yes - 2.3.4 2.3.3 2004 PSF yes - 2.3.5 2.3.4 2005 PSF yes - 2.4 2.3 2004 PSF yes - 2.4.1 2.4 2005 PSF yes - 2.4.2 2.4.1 2005 PSF yes - 2.4.3 2.4.2 2006 PSF yes - 2.4.4 2.4.3 2006 PSF yes - 2.5 2.4 2006 PSF yes - 2.5.1 2.5 2007 PSF yes - 2.5.2 2.5.1 2008 PSF yes - 2.5.3 2.5.2 2008 PSF yes - 2.6 2.5 2008 PSF yes - 2.6.1 2.6 2008 PSF yes - 2.6.2 2.6.1 2009 PSF yes - 2.6.3 2.6.2 2009 PSF yes - 2.6.4 2.6.3 2009 PSF yes - 2.6.5 2.6.4 2010 PSF yes - 2.7 2.6 2010 PSF yes + 2.2 and above 2.1.1 2001-now PSF yes Footnotes: diff --git a/Lib/Cookie.py b/Lib/Cookie.py index 2eda48c0bb1..db32980aca3 100644 --- a/Lib/Cookie.py +++ b/Lib/Cookie.py @@ -238,7 +238,7 @@ class CookieError(Exception): # a two-way quoting algorithm. Any non-text character is translated # into a 4 character sequence: a forward-slash followed by the # three-digit octal equivalent of the character. Any '\' or '"' is -# quoted with a preceeding '\' slash. +# quoted with a preceding '\' slash. # # These are taken from RFC2068 and RFC2109. # _LegalChars is the list of chars which don't require "'s diff --git a/Lib/Queue.py b/Lib/Queue.py index 2db8d769665..00364b39beb 100644 --- a/Lib/Queue.py +++ b/Lib/Queue.py @@ -109,7 +109,7 @@ def put(self, item, block=True, timeout=None): If optional args 'block' is true and 'timeout' is None (the default), block if necessary until a free slot is available. If 'timeout' is - a positive number, it blocks at most 'timeout' seconds and raises + a non-negative number, it blocks at most 'timeout' seconds and raises the Full exception if no free slot was available within that time. Otherwise ('block' is false), put an item on the queue if a free slot is immediately available, else raise the Full exception ('timeout' @@ -125,7 +125,7 @@ def put(self, item, block=True, timeout=None): while self._qsize() == self.maxsize: self.not_full.wait() elif timeout < 0: - raise ValueError("'timeout' must be a positive number") + raise ValueError("'timeout' must be a non-negative number") else: endtime = _time() + timeout while self._qsize() == self.maxsize: @@ -152,7 +152,7 @@ def get(self, block=True, timeout=None): If optional args 'block' is true and 'timeout' is None (the default), block if necessary until an item is available. If 'timeout' is - a positive number, it blocks at most 'timeout' seconds and raises + a non-negative number, it blocks at most 'timeout' seconds and raises the Empty exception if no item was available within that time. Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored @@ -167,7 +167,7 @@ def get(self, block=True, timeout=None): while not self._qsize(): self.not_empty.wait() elif timeout < 0: - raise ValueError("'timeout' must be a positive number") + raise ValueError("'timeout' must be a non-negative number") else: endtime = _time() + timeout while not self._qsize(): diff --git a/Lib/SimpleHTTPServer.py b/Lib/SimpleHTTPServer.py index 3e0334da3b3..1a172968658 100644 --- a/Lib/SimpleHTTPServer.py +++ b/Lib/SimpleHTTPServer.py @@ -149,6 +149,8 @@ def translate_path(self, path): # abandon query parameters path = path.split('?',1)[0] path = path.split('#',1)[0] + # Don't forget explicit trailing slash when normalizing. Issue17324 + trailing_slash = path.rstrip().endswith('/') path = posixpath.normpath(urllib.unquote(path)) words = path.split('/') words = filter(None, words) @@ -158,6 +160,8 @@ def translate_path(self, path): head, word = os.path.split(word) if word in (os.curdir, os.pardir): continue path = os.path.join(path, word) + if trailing_slash: + path += '/' return path def copyfile(self, source, outputfile): diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index 79dbdc9ea3d..42a8ebb406d 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -53,7 +53,7 @@ def _find_executable(executable, path=None): def _read_output(commandstring): - """Output from succesful command execution or None""" + """Output from successful command execution or None""" # Similar to os.popen(commandstring, "r").read(), # but without actually using os.popen because that # function is not usable during python bootstrap. @@ -68,7 +68,7 @@ def _read_output(commandstring): with contextlib.closing(fp) as fp: cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name) - return fp.read().decode('utf-8').strip() if not os.system(cmd) else None + return fp.read().strip() if not os.system(cmd) else None def _find_build_tool(toolname): @@ -152,7 +152,7 @@ def _find_appropriate_compiler(_config_vars): # are not installed. # # Futhermore, the compiler that can be used varies between - # Xcode releases. Upto Xcode 4 it was possible to use 'gcc-4.2' + # Xcode releases. Up to Xcode 4 it was possible to use 'gcc-4.2' # as the compiler, after that 'clang' should be used because # gcc-4.2 is either not present, or a copy of 'llvm-gcc' that # miscompiles Python. @@ -192,7 +192,7 @@ def _find_appropriate_compiler(_config_vars): if cc != oldcc: # Found a replacement compiler. - # Modify config vars using new compiler, if not already explictly + # Modify config vars using new compiler, if not already explicitly # overriden by an env variable, preserving additional arguments. for cv in _COMPILER_CONFIG_VARS: if cv in _config_vars and cv not in os.environ: @@ -274,7 +274,7 @@ def _check_for_unavailable_sdk(_config_vars): # compile an extension using an SDK that is not present # on the current machine it is better to not use an SDK # than to fail. This is particularly important with - # the standalong Command Line Tools alternative to a + # the standalone Command Line Tools alternative to a # full-blown Xcode install since the CLT packages do not # provide SDKs. If the SDK is not present, it is assumed # that the header files and dev libs have been installed @@ -378,7 +378,7 @@ def customize_config_vars(_config_vars): compilers are present, i.e. when installing pure Python dists. Customization of compiler paths and detection of unavailable archs is deferred - until the first extention module build is + until the first extension module build is requested (in distutils.sysconfig.customize_compiler). Currently called from distutils.sysconfig diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 2df30a22ea5..042db6f4f0e 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -222,7 +222,7 @@ def __seqToRE(self, to_convert, directive): """Convert a list to a regex string for matching a directive. Want possible matching values to be from longest to shortest. This - prevents the possibility of a match occuring for a value that also + prevents the possibility of a match occurring for a value that also a substring of a larger value that should have matched (e.g., 'abc' matching when 'abcdef' should have been the match). diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py index 43f9a6e447d..990c3a6bbcd 100644 --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -171,6 +171,12 @@ def __eq__(self, other): return NotImplemented return self.data == set(ref(item) for item in other) + def __ne__(self, other): + opposite = self.__eq__(other) + if opposite is NotImplemented: + return NotImplemented + return not opposite + def symmetric_difference(self, other): newset = self.copy() newset.symmetric_difference_update(other) diff --git a/Lib/aifc.py b/Lib/aifc.py index a0cfe5fc57e..12c665fa8c1 100644 --- a/Lib/aifc.py +++ b/Lib/aifc.py @@ -123,7 +123,7 @@ compression type, and then write audio frames using writeframesraw. When all frames have been written, either call writeframes('') or close() to patch up the sizes in the header. -Marks can be added anytime. If there are any marks, ypu must call +Marks can be added anytime. If there are any marks, you must call close() after all frames have been written. The close() method is called automatically when the class instance is destroyed. @@ -953,23 +953,27 @@ def open(f, mode=None): sys.argv.append('/usr/demos/data/audio/bach.aiff') fn = sys.argv[1] f = open(fn, 'r') - print "Reading", fn - print "nchannels =", f.getnchannels() - print "nframes =", f.getnframes() - print "sampwidth =", f.getsampwidth() - print "framerate =", f.getframerate() - print "comptype =", f.getcomptype() - print "compname =", f.getcompname() - if sys.argv[2:]: - gn = sys.argv[2] - print "Writing", gn - g = open(gn, 'w') - g.setparams(f.getparams()) - while 1: - data = f.readframes(1024) - if not data: - break - g.writeframes(data) - g.close() + try: + print "Reading", fn + print "nchannels =", f.getnchannels() + print "nframes =", f.getnframes() + print "sampwidth =", f.getsampwidth() + print "framerate =", f.getframerate() + print "comptype =", f.getcomptype() + print "compname =", f.getcompname() + if sys.argv[2:]: + gn = sys.argv[2] + print "Writing", gn + g = open(gn, 'w') + try: + g.setparams(f.getparams()) + while 1: + data = f.readframes(1024) + if not data: + break + g.writeframes(data) + finally: + g.close() + print "Done." + finally: f.close() - print "Done." diff --git a/Lib/calendar.py b/Lib/calendar.py index 441b2f576bc..d3bd2362e93 100644 --- a/Lib/calendar.py +++ b/Lib/calendar.py @@ -220,7 +220,7 @@ def monthdayscalendar(self, year, month): def yeardatescalendar(self, year, width=3): """ Return the data for the specified year ready for formatting. The return - value is a list of month rows. Each month row contains upto width months. + value is a list of month rows. Each month row contains up to width months. Each month contains between 4 and 6 weeks and each week contains 1-7 days. Days are datetime.date objects. """ diff --git a/Lib/cgi.py b/Lib/cgi.py index 67079db0d81..64ba6d16caf 100755 --- a/Lib/cgi.py +++ b/Lib/cgi.py @@ -697,6 +697,9 @@ def read_lines_to_outerboundary(self): if not line: self.done = -1 break + if delim == "\r": + line = delim + line + delim = "" if line[:2] == "--" and last_line_lfend: strippedline = line.strip() if strippedline == next: @@ -713,6 +716,12 @@ def read_lines_to_outerboundary(self): delim = "\n" line = line[:-1] last_line_lfend = True + elif line[-1] == "\r": + # We may interrupt \r\n sequences if they span the 2**16 + # byte boundary + delim = "\r" + line = line[:-1] + last_line_lfend = False else: delim = "" last_line_lfend = False diff --git a/Lib/collections.py b/Lib/collections.py index faf677b7bf7..0beb142be59 100644 --- a/Lib/collections.py +++ b/Lib/collections.py @@ -259,8 +259,6 @@ def _asdict(self): 'Return a new OrderedDict which maps field names to their values' return OrderedDict(zip(self._fields, self)) - __dict__ = property(_asdict) - def _replace(_self, **kwds): 'Return a new {typename} object replacing specified fields with new values' result = _self._make(map(kwds.pop, {field_names!r}, _self)) @@ -272,6 +270,12 @@ def __getnewargs__(self): 'Return self as a plain tuple. Used by copy and pickle.' return tuple(self) + __dict__ = _property(_asdict) + + def __getstate__(self): + 'Exclude the OrderedDict from pickling' + pass + {field_defs} ''' @@ -365,7 +369,7 @@ def namedtuple(typename, field_names, verbose=False, rename=False): result = namespace[typename] # For pickling to work, the __module__ variable needs to be set to the frame - # where the named tuple is created. Bypass this step in enviroments where + # where the named tuple is created. Bypass this step in environments where # sys._getframe is not defined (Jython for example) or sys._getframe is not # defined for arguments greater than 0 (IronPython). try: diff --git a/Lib/compiler/pyassem.py b/Lib/compiler/pyassem.py index 286be0c8c7d..f52f7d079f3 100644 --- a/Lib/compiler/pyassem.py +++ b/Lib/compiler/pyassem.py @@ -125,7 +125,7 @@ def order_blocks(start_block, exit_block): # Make sure every block appears in dominators, even if no # other block must precede it. dominators.setdefault(b, set()) - # preceeding blocks dominate following blocks + # preceding blocks dominate following blocks for c in b.get_followers(): while 1: dominators.setdefault(c, set()).add(b) diff --git a/Lib/csv.py b/Lib/csv.py index 984ed7e581b..98480ba16a1 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -261,8 +261,9 @@ def _guess_quote_and_delimiter(self, data, delimiters): # if we see an extra quote between delimiters, we've got a # double quoted format - dq_regexp = re.compile(r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \ - {'delim':delim, 'quote':quotechar}, re.MULTILINE) + dq_regexp = re.compile( + r"((%(delim)s)|^)\W*%(quote)s[^%(delim)s\n]*%(quote)s[^%(delim)s\n]*%(quote)s\W*((%(delim)s)|$)" % \ + {'delim':re.escape(delim), 'quote':quotechar}, re.MULTILINE) diff --git a/Lib/ctypes/test/__init__.py b/Lib/ctypes/test/__init__.py index 52230928ba5..808e4185ef0 100644 --- a/Lib/ctypes/test/__init__.py +++ b/Lib/ctypes/test/__init__.py @@ -62,7 +62,7 @@ def get_tests(package, mask, verbosity, exclude=()): continue try: mod = __import__(modname, globals(), locals(), ['*']) - except ResourceDenied, detail: + except (ResourceDenied, unittest.SkipTest) as detail: skipped.append(modname) if verbosity > 1: print >> sys.stderr, "Skipped %s: %s" % (modname, detail) diff --git a/Lib/ctypes/test/runtests.py b/Lib/ctypes/test/runtests.py index ec31fc831dd..b7a2b264721 100644 --- a/Lib/ctypes/test/runtests.py +++ b/Lib/ctypes/test/runtests.py @@ -2,7 +2,7 @@ Run all tests found in this directory, and print a summary of the results. Command line flags: - -q quiet mode: don't prnt anything while the tests are running + -q quiet mode: don't print anything while the tests are running -r run tests repeatedly, look for refcount leaks -u Add resources to the lits of allowed resources. '*' allows all diff --git a/Lib/ctypes/test/test_cfuncs.py b/Lib/ctypes/test/test_cfuncs.py index 493cbe979ce..598776e0080 100644 --- a/Lib/ctypes/test/test_cfuncs.py +++ b/Lib/ctypes/test/test_cfuncs.py @@ -188,7 +188,7 @@ def test_void(self): self.assertEqual(self._dll.tv_i(-42), None) self.assertEqual(self.S(), -42) -# The following repeates the above tests with stdcall functions (where +# The following repeats the above tests with stdcall functions (where # they are available) try: WinDLL diff --git a/Lib/ctypes/test/test_integers.py b/Lib/ctypes/test/test_integers.py index 5b6453a002a..62e4b08a2b6 100644 --- a/Lib/ctypes/test/test_integers.py +++ b/Lib/ctypes/test/test_integers.py @@ -1,4 +1,4 @@ -# superseeded by test_numbers.py +# superseded by test_numbers.py import unittest if __name__ == '__main__': diff --git a/Lib/ctypes/test/test_numbers.py b/Lib/ctypes/test/test_numbers.py index 79239068306..555e04a4a9b 100644 --- a/Lib/ctypes/test/test_numbers.py +++ b/Lib/ctypes/test/test_numbers.py @@ -212,7 +212,7 @@ def test_char_from_address(self): def test_init(self): # c_int() can be initialized from Python's int, and c_int. - # Not from c_long or so, which seems strange, abd should + # Not from c_long or so, which seems strange, abc should # probably be changed: self.assertRaises(TypeError, c_int, c_long(42)) diff --git a/Lib/ctypes/test/test_refcounts.py b/Lib/ctypes/test/test_refcounts.py index 35a81aa40be..fe4c4b92596 100644 --- a/Lib/ctypes/test/test_refcounts.py +++ b/Lib/ctypes/test/test_refcounts.py @@ -41,7 +41,7 @@ def func(*args): # this is the standard refcount for func self.assertEqual(grc(func), 2) - # the CFuncPtr instance holds atr least one refcount on func: + # the CFuncPtr instance holds at least one refcount on func: f = OtherCallback(func) self.assertTrue(grc(func) > 2) @@ -58,7 +58,7 @@ class X(ctypes.Structure): x = X() x.a = OtherCallback(func) - # the CFuncPtr instance holds atr least one refcount on func: + # the CFuncPtr instance holds at least one refcount on func: self.assertTrue(grc(func) > 2) # and may release it again @@ -71,7 +71,7 @@ class X(ctypes.Structure): f = OtherCallback(func) - # the CFuncPtr instance holds atr least one refcount on func: + # the CFuncPtr instance holds at least one refcount on func: self.assertTrue(grc(func) > 2) # create a cycle diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py index 2d7c816b241..a3350ce7774 100644 --- a/Lib/ctypes/test/test_structures.py +++ b/Lib/ctypes/test/test_structures.py @@ -108,7 +108,7 @@ class XX(Structure): def test_emtpy(self): # I had problems with these # - # Although these are patological cases: Empty Structures! + # Although these are pathological cases: Empty Structures! class X(Structure): _fields_ = [] diff --git a/Lib/ctypes/test/test_wintypes.py b/Lib/ctypes/test/test_wintypes.py new file mode 100644 index 00000000000..806fccef812 --- /dev/null +++ b/Lib/ctypes/test/test_wintypes.py @@ -0,0 +1,43 @@ +import sys +import unittest + +if not sys.platform.startswith('win'): + raise unittest.SkipTest('Windows-only test') + +from ctypes import * +from ctypes import wintypes + +class WinTypesTest(unittest.TestCase): + def test_variant_bool(self): + # reads 16-bits from memory, anything non-zero is True + for true_value in (1, 32767, 32768, 65535, 65537): + true = POINTER(c_int16)(c_int16(true_value)) + value = cast(true, POINTER(wintypes.VARIANT_BOOL)) + self.assertEqual(repr(value.contents), 'VARIANT_BOOL(True)') + + vb = wintypes.VARIANT_BOOL() + self.assertIs(vb.value, False) + vb.value = True + self.assertIs(vb.value, True) + vb.value = true_value + self.assertIs(vb.value, True) + + for false_value in (0, 65536, 262144, 2**33): + false = POINTER(c_int16)(c_int16(false_value)) + value = cast(false, POINTER(wintypes.VARIANT_BOOL)) + self.assertEqual(repr(value.contents), 'VARIANT_BOOL(False)') + + # allow any bool conversion on assignment to value + for set_value in (65536, 262144, 2**33): + vb = wintypes.VARIANT_BOOL() + vb.value = set_value + self.assertIs(vb.value, True) + + vb = wintypes.VARIANT_BOOL() + vb.value = [2, 3] + self.assertIs(vb.value, True) + vb.value = [] + self.assertIs(vb.value, False) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index fe2b5205325..fe0ed0a085f 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -93,7 +93,7 @@ def _findLib_gcc(name): fdout, ccout = tempfile.mkstemp() os.close(fdout) cmd = 'if type gcc >/dev/null 2>&1; then CC=gcc; elif type cc >/dev/null 2>&1; then CC=cc;else exit 10; fi;' \ - '$CC -Wl,-t -o ' + ccout + ' 2>&1 -l' + name + 'LANG=C LC_ALL=C $CC -Wl,-t -o ' + ccout + ' 2>&1 -l' + name try: f = os.popen(cmd) try: diff --git a/Lib/decimal.py b/Lib/decimal.py index 361126f6d0e..04bf5c26605 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -25,7 +25,7 @@ and IEEE standard 854-1987: - www.cs.berkeley.edu/~ejr/projects/754/private/drafts/854-1987/dir.html + http://en.wikipedia.org/wiki/IEEE_854-1987 Decimal floating point has finite precision with arbitrarily large bounds. diff --git a/Lib/difflib.py b/Lib/difflib.py index 3bbcb76b7ec..f94b3aca29b 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -586,7 +586,7 @@ def get_opcodes(self): def get_grouped_opcodes(self, n=3): """ Isolate change clusters by eliminating ranges with no changes. - Return a generator of groups with upto n lines of context. + Return a generator of groups with up to n lines of context. Each group is in the same format as returned by get_opcodes(). >>> from pprint import pprint @@ -1361,7 +1361,7 @@ def _mdiff(fromlines, tolines, context=None, linejunk=None, linejunk -- passed on to ndiff (see ndiff documentation) charjunk -- passed on to ndiff (see ndiff documentation) - This function returns an interator which returns a tuple: + This function returns an iterator which returns a tuple: (from line tuple, to line tuple, boolean flag) from/to line tuple -- (line num, line text) @@ -1963,7 +1963,7 @@ def make_table(self,fromlines,tolines,fromdesc='',todesc='',context=False, self._make_prefix() # change tabs to spaces before it gets more difficult after we insert - # markkup + # markup fromlines,tolines = self._tab_newline_replace(fromlines,tolines) # create diffs iterator which generates side by side from/to data diff --git a/Lib/distutils/__init__.py b/Lib/distutils/__init__.py index 15b511bee58..98f4bb3a49a 100644 --- a/Lib/distutils/__init__.py +++ b/Lib/distutils/__init__.py @@ -15,5 +15,5 @@ # Updated automatically by the Python release process. # #--start constants-- -__version__ = "2.7.4rc1" +__version__ = "2.7.5" #--end constants-- diff --git a/Lib/distutils/command/build_ext.py b/Lib/distutils/command/build_ext.py index 923197bac43..f0a7d4cafdd 100644 --- a/Lib/distutils/command/build_ext.py +++ b/Lib/distutils/command/build_ext.py @@ -231,12 +231,10 @@ def finalize_options(self): # building python standard extensions self.library_dirs.append('.') - # for extensions under Linux or Solaris with a shared Python library, + # For building extensions with a shared Python library, # Python's library directory must be appended to library_dirs - sysconfig.get_config_var('Py_ENABLE_SHARED') - if ((sys.platform.startswith('linux') or sys.platform.startswith('gnu') - or sys.platform.startswith('sunos')) - and sysconfig.get_config_var('Py_ENABLE_SHARED')): + # See Issues: #1600860, #4366 + if (sysconfig.get_config_var('Py_ENABLE_SHARED')): if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): # building third party extensions self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) diff --git a/Lib/distutils/command/sdist.py b/Lib/distutils/command/sdist.py index d30de10673c..821420d62b8 100644 --- a/Lib/distutils/command/sdist.py +++ b/Lib/distutils/command/sdist.py @@ -183,7 +183,7 @@ def get_file_list(self): depends on the user's options. """ # new behavior when using a template: - # the file list is recalculated everytime because + # the file list is recalculated every time because # even if MANIFEST.in or setup.py are not changed # the user might have added some files in the tree that # need to be included. diff --git a/Lib/distutils/cygwinccompiler.py b/Lib/distutils/cygwinccompiler.py index a1ee815c6cc..5d116876a33 100644 --- a/Lib/distutils/cygwinccompiler.py +++ b/Lib/distutils/cygwinccompiler.py @@ -319,13 +319,18 @@ def __init__ (self, else: entry_point = '' - self.set_executables(compiler='gcc -mno-cygwin -O -Wall', - compiler_so='gcc -mno-cygwin -mdll -O -Wall', - compiler_cxx='g++ -mno-cygwin -O -Wall', - linker_exe='gcc -mno-cygwin', - linker_so='%s -mno-cygwin %s %s' - % (self.linker_dll, shared_option, - entry_point)) + if self.gcc_version < '4' or is_cygwingcc(): + no_cygwin = ' -mno-cygwin' + else: + no_cygwin = '' + + self.set_executables(compiler='gcc%s -O -Wall' % no_cygwin, + compiler_so='gcc%s -mdll -O -Wall' % no_cygwin, + compiler_cxx='g++%s -O -Wall' % no_cygwin, + linker_exe='gcc%s' % no_cygwin, + linker_so='%s%s %s %s' + % (self.linker_dll, no_cygwin, + shared_option, entry_point)) # Maybe we should also append -mthreads, but then the finished # dlls need another dll (mingwm10.dll see Mingw32 docs) # (-mthreads: Support thread-safe exception handling on `Mingw32') @@ -447,3 +452,12 @@ def get_versions(): else: dllwrap_version = None return (gcc_version, ld_version, dllwrap_version) + +def is_cygwingcc(): + '''Try to determine if the gcc that would be used is from cygwin.''' + out = os.popen('gcc -dumpmachine', 'r') + out_string = out.read() + out.close() + # out_string is the target triplet cpu-vendor-os + # Cygwin's gcc sets the os to 'cygwin' + return out_string.strip().endswith('cygwin') diff --git a/Lib/distutils/sysconfig.py b/Lib/distutils/sysconfig.py index 250ef38bedf..4aa93346d53 100644 --- a/Lib/distutils/sysconfig.py +++ b/Lib/distutils/sysconfig.py @@ -175,9 +175,15 @@ def customize_compiler(compiler): 'CCSHARED', 'LDSHARED', 'SO', 'AR', 'ARFLAGS') - newcc = None if 'CC' in os.environ: - cc = os.environ['CC'] + newcc = os.environ['CC'] + if (sys.platform == 'darwin' + and 'LDSHARED' not in os.environ + and ldshared.startswith(cc)): + # On OS X, if CC is overridden, use that as the default + # command for LDSHARED as well + ldshared = newcc + ldshared[len(cc):] + cc = newcc if 'CXX' in os.environ: cxx = os.environ['CXX'] if 'LDSHARED' in os.environ: @@ -387,66 +393,11 @@ def expand_makefile_vars(s, vars): def _init_posix(): """Initialize the module as appropriate for POSIX systems.""" - g = {} - # load the installed Makefile: - try: - filename = get_makefile_filename() - parse_makefile(filename, g) - except IOError, msg: - my_msg = "invalid Python installation: unable to open %s" % filename - if hasattr(msg, "strerror"): - my_msg = my_msg + " (%s)" % msg.strerror - - raise DistutilsPlatformError(my_msg) - - # load the installed pyconfig.h: - try: - filename = get_config_h_filename() - parse_config_h(file(filename), g) - except IOError, msg: - my_msg = "invalid Python installation: unable to open %s" % filename - if hasattr(msg, "strerror"): - my_msg = my_msg + " (%s)" % msg.strerror - - raise DistutilsPlatformError(my_msg) - - # On AIX, there are wrong paths to the linker scripts in the Makefile - # -- these paths are relative to the Python source, but when installed - # the scripts are in another directory. - if python_build: - g['LDSHARED'] = g['BLDSHARED'] - - elif get_python_version() < '2.1': - # The following two branches are for 1.5.2 compatibility. - if sys.platform == 'aix4': # what about AIX 3.x ? - # Linker script is in the config directory, not in Modules as the - # Makefile says. - python_lib = get_python_lib(standard_lib=1) - ld_so_aix = os.path.join(python_lib, 'config', 'ld_so_aix') - python_exp = os.path.join(python_lib, 'config', 'python.exp') - - g['LDSHARED'] = "%s %s -bI:%s" % (ld_so_aix, g['CC'], python_exp) - - elif sys.platform == 'beos': - # Linker script is in the config directory. In the Makefile it is - # relative to the srcdir, which after installation no longer makes - # sense. - python_lib = get_python_lib(standard_lib=1) - linkerscript_path = string.split(g['LDSHARED'])[0] - linkerscript_name = os.path.basename(linkerscript_path) - linkerscript = os.path.join(python_lib, 'config', - linkerscript_name) - - # XXX this isn't the right place to do this: adding the Python - # library to the link, if needed, should be in the "build_ext" - # command. (It's also needed for non-MS compilers on Windows, and - # it's taken care of for them by the 'build_ext.get_libraries()' - # method.) - g['LDSHARED'] = ("%s -L%s/lib -lpython%s" % - (linkerscript, PREFIX, get_python_version())) - + # _sysconfigdata is generated at build time, see the sysconfig module + from _sysconfigdata import build_time_vars global _config_vars - _config_vars = g + _config_vars = {} + _config_vars.update(build_time_vars) def _init_nt(): diff --git a/Lib/distutils/tests/test_build_clib.py b/Lib/distutils/tests/test_build_clib.py index bef1bd99536..12faa8a9548 100644 --- a/Lib/distutils/tests/test_build_clib.py +++ b/Lib/distutils/tests/test_build_clib.py @@ -77,7 +77,7 @@ def compile(*args, **kw): cmd.compiler = FakeCompiler() - # build_libraries is also doing a bit of typoe checking + # build_libraries is also doing a bit of typo checking lib = [('name', {'sources': 'notvalid'})] self.assertRaises(DistutilsSetupError, cmd.build_libraries, lib) diff --git a/Lib/distutils/tests/test_cmd.py b/Lib/distutils/tests/test_cmd.py index e074099609a..51420c96cc0 100644 --- a/Lib/distutils/tests/test_cmd.py +++ b/Lib/distutils/tests/test_cmd.py @@ -34,6 +34,18 @@ def test_ensure_string_list(self): self.assertRaises(DistutilsOptionError, cmd.ensure_string_list, 'not_string_list2') + cmd.option1 = 'ok,dok' + cmd.ensure_string_list('option1') + self.assertEqual(cmd.option1, ['ok', 'dok']) + + cmd.option2 = ['xxx', 'www'] + cmd.ensure_string_list('option2') + + cmd.option3 = ['ok', 2] + self.assertRaises(DistutilsOptionError, cmd.ensure_string_list, + 'option3') + + def test_make_file(self): cmd = self.cmd @@ -77,19 +89,6 @@ def test_ensure_string(self): cmd.option3 = 1 self.assertRaises(DistutilsOptionError, cmd.ensure_string, 'option3') - def test_ensure_string_list(self): - cmd = self.cmd - cmd.option1 = 'ok,dok' - cmd.ensure_string_list('option1') - self.assertEqual(cmd.option1, ['ok', 'dok']) - - cmd.option2 = ['xxx', 'www'] - cmd.ensure_string_list('option2') - - cmd.option3 = ['ok', 2] - self.assertRaises(DistutilsOptionError, cmd.ensure_string_list, - 'option3') - def test_ensure_filename(self): cmd = self.cmd cmd.option1 = __file__ diff --git a/Lib/distutils/tests/test_unixccompiler.py b/Lib/distutils/tests/test_unixccompiler.py index 40c908a24d0..64f9d267511 100644 --- a/Lib/distutils/tests/test_unixccompiler.py +++ b/Lib/distutils/tests/test_unixccompiler.py @@ -1,7 +1,8 @@ """Tests for distutils.unixccompiler.""" +import os import sys import unittest -from test.test_support import run_unittest +from test.test_support import EnvironmentVarGuard, run_unittest from distutils import sysconfig from distutils.unixccompiler import UnixCCompiler @@ -122,6 +123,37 @@ def gcv(v): sysconfig.get_config_var = gcv self.assertEqual(self.cc.rpath_foo(), '-R/foo') + @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for OS X') + def test_osx_cc_overrides_ldshared(self): + # Issue #18080: + # ensure that setting CC env variable also changes default linker + def gcv(v): + if v == 'LDSHARED': + return 'gcc-4.2 -bundle -undefined dynamic_lookup ' + return 'gcc-4.2' + sysconfig.get_config_var = gcv + with EnvironmentVarGuard() as env: + env['CC'] = 'my_cc' + del env['LDSHARED'] + sysconfig.customize_compiler(self.cc) + self.assertEqual(self.cc.linker_so[0], 'my_cc') + + @unittest.skipUnless(sys.platform == 'darwin', 'test only relevant for OS X') + def test_osx_explict_ldshared(self): + # Issue #18080: + # ensure that setting CC env variable does not change + # explicit LDSHARED setting for linker + def gcv(v): + if v == 'LDSHARED': + return 'gcc-4.2 -bundle -undefined dynamic_lookup ' + return 'gcc-4.2' + sysconfig.get_config_var = gcv + with EnvironmentVarGuard() as env: + env['CC'] = 'my_cc' + env['LDSHARED'] = 'my_ld -bundle -dynamic' + sysconfig.customize_compiler(self.cc) + self.assertEqual(self.cc.linker_so[0], 'my_ld') + def test_suite(): return unittest.makeSuite(UnixCCompilerTestCase) diff --git a/Lib/doctest.py b/Lib/doctest.py index 90bcca1fdbe..efa07d7ea51 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -424,7 +424,7 @@ class Example: zero-based, with respect to the beginning of the DocTest. - indent: The example's indentation in the DocTest string. - I.e., the number of space characters that preceed the + I.e., the number of space characters that precede the example's first prompt. - options: A dictionary mapping from option flags to True or @@ -564,7 +564,7 @@ class DocTestParser: # Want consists of any non-blank lines that do not start with PS1. (?P (?:(?![ ]*$) # Not a blank line (?![ ]*>>>) # Not a line starting with PS1 - .*$\n? # But any other line + .+$\n? # But any other line )*) ''', re.MULTILINE | re.VERBOSE) @@ -895,7 +895,7 @@ def find(self, obj, name=None, module=None, globs=None, extraglobs=None): if '__name__' not in globs: globs['__name__'] = '__main__' # provide a default module name - # Recursively expore `obj`, extracting DocTests. + # Recursively explore `obj`, extracting DocTests. tests = [] self._find(tests, obj, name, module, source_lines, globs, {}) # Sort the tests by alpha order of names, for consistency in diff --git a/Lib/email/charset.py b/Lib/email/charset.py index dddaa76c55d..30a13ff2494 100644 --- a/Lib/email/charset.py +++ b/Lib/email/charset.py @@ -183,7 +183,7 @@ class Charset: header encoding. Charset.SHORTEST is not allowed for body_encoding. - output_charset: Some character sets must be converted before the can be + output_charset: Some character sets must be converted before they can be used in email headers or bodies. If the input_charset is one of them, this attribute will contain the name of the charset output will be converted to. Otherwise, it will diff --git a/Lib/filecmp.py b/Lib/filecmp.py index 4728317fce9..3a793819c64 100644 --- a/Lib/filecmp.py +++ b/Lib/filecmp.py @@ -268,7 +268,7 @@ def cmpfiles(a, b, common, shallow=1): def _cmp(a, b, sh, abs=abs, cmp=cmp): try: return not abs(cmp(a, b, sh)) - except os.error: + except (os.error, IOError): return 2 diff --git a/Lib/fileinput.py b/Lib/fileinput.py index ba485752092..04e97bdb418 100644 --- a/Lib/fileinput.py +++ b/Lib/fileinput.py @@ -90,12 +90,11 @@ def input(files=None, inplace=0, backup="", bufsize=0, mode="r", openhook=None): - """input([files[, inplace[, backup[, mode[, openhook]]]]]) + """Return an instance of the FileInput class, which can be iterated. - Create an instance of the FileInput class. The instance will be used - as global state for the functions of this module, and is also returned - to use during iteration. The parameters to this function will be passed - along to the constructor of the FileInput class. + The parameters are passed to the constructor of the FileInput class. + The returned instance, in addition to being an iterator, + keeps global state for the functions of this module,. """ global _state if _state and _state._file: @@ -182,7 +181,7 @@ def isstdin(): return _state.isstdin() class FileInput: - """class FileInput([files[, inplace[, backup[, mode[, openhook]]]]]) + """FileInput([files[, inplace[, backup[, bufsize[, mode[, openhook]]]]]]) Class FileInput is the implementation of the module; its methods filename(), lineno(), fileline(), isfirstline(), isstdin(), fileno(), diff --git a/Lib/genericpath.py b/Lib/genericpath.py index a0bf6013e93..7ddb94c08b9 100644 --- a/Lib/genericpath.py +++ b/Lib/genericpath.py @@ -22,7 +22,7 @@ def exists(path): # This follows symbolic links, so both islink() and isdir() can be true -# for the same path ono systems that support symlinks +# for the same path on systems that support symlinks def isfile(path): """Test whether a path is a regular file""" try: diff --git a/Lib/gzip.py b/Lib/gzip.py index a2f23679fa8..a613bae876d 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -21,6 +21,9 @@ def write32u(output, value): # or unsigned. output.write(struct.pack(" self.extrasize: - if not self._read(readsize): - if size > self.extrasize: - size = self.extrasize - break - readsize = min(self.max_read_chunk, readsize * 2) + try: + while size > self.extrasize: + self._read(readsize) + readsize = min(self.max_read_chunk, readsize * 2) + except EOFError: + if size > self.extrasize: + size = self.extrasize offset = self.offset - self.extrastart chunk = self.extrabuf[offset: offset + size] @@ -274,7 +277,7 @@ def _unread(self, buf): def _read(self, size=1024): if self.fileobj is None: - return False + raise EOFError, "Reached EOF" if self._new_member: # If the _new_member flag is set, we have to @@ -285,7 +288,7 @@ def _read(self, size=1024): pos = self.fileobj.tell() # Save current position self.fileobj.seek(0, 2) # Seek to end of file if pos == self.fileobj.tell(): - return False + raise EOFError, "Reached EOF" else: self.fileobj.seek( pos ) # Return to original position @@ -302,10 +305,9 @@ def _read(self, size=1024): if buf == "": uncompress = self.decompress.flush() - self.fileobj.seek(-len(self.decompress.unused_data), 1) self._read_eof() self._add_read_data( uncompress ) - return False + raise EOFError, 'Reached EOF' uncompress = self.decompress.decompress(buf) self._add_read_data( uncompress ) @@ -315,14 +317,13 @@ def _read(self, size=1024): # so seek back to the start of the unused data, finish up # this member, and read a new gzip header. # (The number of bytes to seek back is the length of the unused - # data) - self.fileobj.seek(-len(self.decompress.unused_data), 1) + # data, minus 8 because _read_eof() will rewind a further 8 bytes) + self.fileobj.seek( -len(self.decompress.unused_data)+8, 1) # Check the CRC and file size, and set the flag so we read # a new member on the next call self._read_eof() self._new_member = True - return True def _add_read_data(self, data): self.crc = zlib.crc32(data, self.crc) & 0xffffffffL @@ -333,11 +334,14 @@ def _add_read_data(self, data): self.size = self.size + len(data) def _read_eof(self): - # We've read to the end of the file. + # We've read to the end of the file, so we have to rewind in order + # to reread the 8 bytes containing the CRC and the file size. # We check the that the computed CRC and size of the # uncompressed data matches the stored values. Note that the size # stored is the true file size mod 2**32. - crc32, isize = struct.unpack(" 1: try: - while 1: - v, itnum, next = s = h[0] # raises IndexError when h is empty + while True: + v, itnum, next = s = h[0] yield v s[0] = next() # raises StopIteration when exhausted _heapreplace(h, s) # restore heap condition except _StopIteration: _heappop(h) # remove empty iterator - except IndexError: - return + if h: + # fast case when only a single iterator remains + v, itnum, next = h[0] + yield v + for v in next.__self__: + yield v # Extend the implementations of nsmallest and nlargest to use a key= argument _nsmallest = nsmallest diff --git a/Lib/idlelib/AutoComplete.py b/Lib/idlelib/AutoComplete.py index 4e173252d61..cb38691097e 100644 --- a/Lib/idlelib/AutoComplete.py +++ b/Lib/idlelib/AutoComplete.py @@ -156,12 +156,9 @@ def open_completions(self, evalfuncs, complete, userWantsWin, mode=None): if not comp_lists[0]: return self.autocompletewindow = self._make_autocomplete_window() - self.autocompletewindow.show_window(comp_lists, - "insert-%dc" % len(comp_start), - complete, - mode, - userWantsWin) - return True + return not self.autocompletewindow.show_window( + comp_lists, "insert-%dc" % len(comp_start), + complete, mode, userWantsWin) def fetch_completions(self, what, mode): """Return a pair of lists of completions for something. The first list diff --git a/Lib/idlelib/AutoCompleteWindow.py b/Lib/idlelib/AutoCompleteWindow.py index 298177fc847..27b0e56bd58 100644 --- a/Lib/idlelib/AutoCompleteWindow.py +++ b/Lib/idlelib/AutoCompleteWindow.py @@ -157,13 +157,14 @@ def show_window(self, comp_lists, index, complete, mode, userWantsWin): self.start = self.widget.get(self.startindex, "insert") if complete: completed = self._complete_string(self.start) + start = self.start self._change_start(completed) i = self._binary_search(completed) if self.completions[i] == completed and \ (i == len(self.completions)-1 or self.completions[i+1][:len(completed)] != completed): # There is exactly one matching completion - return + return completed == start self.userwantswindow = userWantsWin self.lasttypedstart = self.start diff --git a/Lib/idlelib/Bindings.py b/Lib/idlelib/Bindings.py index ec2720b0d02..65c0317e60c 100644 --- a/Lib/idlelib/Bindings.py +++ b/Lib/idlelib/Bindings.py @@ -15,7 +15,7 @@ menudefs = [ # underscore prefixes character to underscore ('file', [ - ('_New Window', '<>'), + ('_New File', '<>'), ('_Open...', '<>'), ('Open _Module...', '<>'), ('Class _Browser', '<>'), @@ -98,6 +98,10 @@ # menu del menudefs[-1][1][0:2] + # Remove the 'Configure' entry from the options menu, it is in the + # application menu as 'Preferences' + del menudefs[-2][1][0:2] + default_keydefs = idleConf.GetCurrentKeySet() del sys diff --git a/Lib/idlelib/CallTips.py b/Lib/idlelib/CallTips.py index d533ce13b95..c29f89bc0dd 100644 --- a/Lib/idlelib/CallTips.py +++ b/Lib/idlelib/CallTips.py @@ -163,7 +163,7 @@ def get_arg_text(ob): if fob.func_code.co_flags & 0x8: items.append("***") arg_text = ", ".join(items) - arg_text = "(%s)" % re.sub("\.\d+", "", arg_text) + arg_text = "(%s)" % re.sub("(?", arg_text) # See if we can use the docstring doc = getattr(ob, "__doc__", "") if doc: @@ -223,4 +223,6 @@ def test(tests): tests = (t1, t2, t3, t4, t5, t6, t7, TC, tc.t1, tc.t2, tc.t3, tc.t4, tc.t5, tc.t6, tc.t7) - test(tests) + # test(tests) + from unittest import main + main('idlelib.idle_test.test_calltips', verbosity=2, exit=False) diff --git a/Lib/idlelib/ColorDelegator.py b/Lib/idlelib/ColorDelegator.py index 0610c4b6aac..c74865d4fb4 100644 --- a/Lib/idlelib/ColorDelegator.py +++ b/Lib/idlelib/ColorDelegator.py @@ -50,6 +50,10 @@ def setdelegate(self, delegate): self.config_colors() self.bind("<>", self.toggle_colorize_event) self.notify_range("1.0", "end") + else: + # No delegate - stop any colorizing + self.stop_colorizing = True + self.allow_colorizing = False def config_colors(self): for tag, cnf in self.tagdefs.items(): diff --git a/Lib/idlelib/Delegator.py b/Lib/idlelib/Delegator.py index 6125591fe0d..c4765163f80 100644 --- a/Lib/idlelib/Delegator.py +++ b/Lib/idlelib/Delegator.py @@ -4,30 +4,22 @@ class Delegator: def __init__(self, delegate=None): self.delegate = delegate - self.__cache = {} + self.__cache = set() def __getattr__(self, name): attr = getattr(self.delegate, name) # May raise AttributeError setattr(self, name, attr) - self.__cache[name] = attr + self.__cache.add(name) return attr def resetcache(self): - for key in self.__cache.keys(): + for key in self.__cache: try: delattr(self, key) except AttributeError: pass self.__cache.clear() - def cachereport(self): - keys = self.__cache.keys() - keys.sort() - print keys - def setdelegate(self, delegate): self.resetcache() self.delegate = delegate - - def getdelegate(self): - return self.delegate diff --git a/Lib/idlelib/EditorWindow.py b/Lib/idlelib/EditorWindow.py index 0a01c9ec305..07ca5562330 100644 --- a/Lib/idlelib/EditorWindow.py +++ b/Lib/idlelib/EditorWindow.py @@ -346,6 +346,36 @@ def __init__(self, flist=None, filename=None, key=None, root=None): self.askinteger = tkSimpleDialog.askinteger self.showerror = tkMessageBox.showerror + self._highlight_workaround() # Fix selection tags on Windows + + def _highlight_workaround(self): + # On Windows, Tk removes painting of the selection + # tags which is different behavior than on Linux and Mac. + # See issue14146 for more information. + if not sys.platform.startswith('win'): + return + + text = self.text + text.event_add("<>", "") + text.event_add("<>", "") + def highlight_fix(focus): + sel_range = text.tag_ranges("sel") + if sel_range: + if focus == 'out': + HILITE_CONFIG = idleConf.GetHighlight( + idleConf.CurrentTheme(), 'hilite') + text.tag_config("sel_fix", HILITE_CONFIG) + text.tag_raise("sel_fix") + text.tag_add("sel_fix", *sel_range) + elif focus == 'in': + text.tag_remove("sel_fix", "1.0", "end") + + text.bind("<>", + lambda ev: highlight_fix("out")) + text.bind("<>", + lambda ev: highlight_fix("in")) + + def _filename_to_unicode(self, filename): """convert filename to unicode in order to display it in Tk""" if isinstance(filename, unicode) or not filename: @@ -437,7 +467,6 @@ def set_line_and_column(self, event=None): ] if macosxSupport.runningAsOSXApp(): - del menu_specs[-3] menu_specs[-2] = ("windows", "_Window") @@ -480,7 +509,12 @@ def right_menu_event(self, event): if iswin: self.text.config(cursor="arrow") - for label, eventname, verify_state in self.rmenu_specs: + for item in self.rmenu_specs: + try: + label, eventname, verify_state = item + except ValueError: # see issue1207589 + continue + if verify_state is None: continue state = getattr(self, verify_state)() @@ -497,7 +531,8 @@ def right_menu_event(self, event): def make_rmenu(self): rmenu = Menu(self.text, tearoff=0) - for label, eventname, _ in self.rmenu_specs: + for item in self.rmenu_specs: + label, eventname = item[0], item[1] if label is not None: def command(text=self.text, eventname=eventname): text.event_generate(eventname) @@ -654,7 +689,7 @@ def open_module(self, event=None): # XXX Ought to insert current file's directory in front of path try: (f, file, (suffix, mode, type)) = _find_module(name) - except (NameError, ImportError), msg: + except (NameError, ImportError) as msg: tkMessageBox.showerror("Import error", str(msg), parent=self.text) return if type != imp.PY_SOURCE: @@ -798,7 +833,11 @@ def ApplyKeybindings(self): menuEventDict[menu[0]][prepstr(item[0])[1]] = item[1] for menubarItem in self.menudict.keys(): menu = self.menudict[menubarItem] - end = menu.index(END) + 1 + end = menu.index(END) + if end is None: + # Skip empty menus + continue + end += 1 for index in range(0, end): if menu.type(index) == 'command': accel = menu.entrycget(index, 'accelerator') @@ -855,11 +894,8 @@ def update_recent_files_list(self, new_file=None): "Load and update the recent files list and menus" rf_list = [] if os.path.exists(self.recent_files_path): - rf_list_file = open(self.recent_files_path,'r') - try: + with open(self.recent_files_path, 'r') as rf_list_file: rf_list = rf_list_file.readlines() - finally: - rf_list_file.close() if new_file: new_file = os.path.abspath(new_file) + '\n' if new_file in rf_list: @@ -1425,6 +1461,7 @@ def uncomment_region_event(self, event): def tabify_region_event(self, event): head, tail, chars, lines = self.get_region() tabwidth = self._asktabwidth() + if tabwidth is None: return for pos in range(len(lines)): line = lines[pos] if line: @@ -1436,6 +1473,7 @@ def tabify_region_event(self, event): def untabify_region_event(self, event): head, tail, chars, lines = self.get_region() tabwidth = self._asktabwidth() + if tabwidth is None: return for pos in range(len(lines)): lines[pos] = lines[pos].expandtabs(tabwidth) self.set_region(head, tail, chars, lines) @@ -1529,7 +1567,7 @@ def _asktabwidth(self): parent=self.text, initialvalue=self.indentwidth, minvalue=2, - maxvalue=16) or self.tabwidth + maxvalue=16) # Guess indentwidth from text content. # Return guessed indentwidth. This should not be believed unless diff --git a/Lib/idlelib/FormatParagraph.py b/Lib/idlelib/FormatParagraph.py index 557d8a9e0d7..ae4e6e7b911 100644 --- a/Lib/idlelib/FormatParagraph.py +++ b/Lib/idlelib/FormatParagraph.py @@ -1,18 +1,19 @@ -# Extension to format a paragraph - -# Does basic, standard text formatting, and also understands Python -# comment blocks. Thus, for editing Python source code, this -# extension is really only suitable for reformatting these comment -# blocks or triple-quoted strings. - -# Known problems with comment reformatting: -# * If there is a selection marked, and the first line of the -# selection is not complete, the block will probably not be detected -# as comments, and will have the normal "text formatting" rules -# applied. -# * If a comment block has leading whitespace that mixes tabs and -# spaces, they will not be considered part of the same block. -# * Fancy comments, like this bulleted list, arent handled :-) +"""Extension to format a paragraph or selection to a max width. + +Does basic, standard text formatting, and also understands Python +comment blocks. Thus, for editing Python source code, this +extension is really only suitable for reformatting these comment +blocks or triple-quoted strings. + +Known problems with comment reformatting: +* If there is a selection marked, and the first line of the + selection is not complete, the block will probably not be detected + as comments, and will have the normal "text formatting" rules + applied. +* If a comment block has leading whitespace that mixes tabs and + spaces, they will not be considered part of the same block. +* Fancy comments, like this bulleted list, aren't handled :-) +""" import re from idlelib.configHandler import idleConf @@ -32,41 +33,31 @@ def close(self): self.editwin = None def format_paragraph_event(self, event): - maxformatwidth = int(idleConf.GetOption('main','FormatParagraph', - 'paragraph', type='int')) + """Formats paragraph to a max width specified in idleConf. + + If text is selected, format_paragraph_event will start breaking lines + at the max width, starting from the beginning selection. + + If no text is selected, format_paragraph_event uses the current + cursor location to determine the paragraph (lines of text surrounded + by blank lines) and formats it. + """ + maxformatwidth = idleConf.GetOption( + 'main', 'FormatParagraph', 'paragraph', type='int') text = self.editwin.text first, last = self.editwin.get_selection_indices() if first and last: data = text.get(first, last) - comment_header = '' + comment_header = get_comment_header(data) else: first, last, comment_header, data = \ find_paragraph(text, text.index("insert")) if comment_header: - # Reformat the comment lines - convert to text sans header. - lines = data.split("\n") - lines = map(lambda st, l=len(comment_header): st[l:], lines) - data = "\n".join(lines) - # Reformat to maxformatwidth chars or a 20 char width, whichever is greater. - format_width = max(maxformatwidth - len(comment_header), 20) - newdata = reformat_paragraph(data, format_width) - # re-split and re-insert the comment header. - newdata = newdata.split("\n") - # If the block ends in a \n, we dont want the comment - # prefix inserted after it. (Im not sure it makes sense to - # reformat a comment block that isnt made of complete - # lines, but whatever!) Can't think of a clean solution, - # so we hack away - block_suffix = "" - if not newdata[-1]: - block_suffix = "\n" - newdata = newdata[:-1] - builder = lambda item, prefix=comment_header: prefix+item - newdata = '\n'.join(map(builder, newdata)) + block_suffix + newdata = reformat_comment(data, maxformatwidth, comment_header) else: - # Just a normal text format newdata = reformat_paragraph(data, maxformatwidth) text.tag_remove("sel", "1.0", "end") + if newdata != data: text.mark_set("insert", first) text.undo_block_start() @@ -79,31 +70,44 @@ def format_paragraph_event(self, event): return "break" def find_paragraph(text, mark): + """Returns the start/stop indices enclosing the paragraph that mark is in. + + Also returns the comment format string, if any, and paragraph of text + between the start/stop indices. + """ lineno, col = map(int, mark.split(".")) - line = text.get("%d.0" % lineno, "%d.0 lineend" % lineno) + line = text.get("%d.0" % lineno, "%d.end" % lineno) + + # Look for start of next paragraph if the index passed in is a blank line while text.compare("%d.0" % lineno, "<", "end") and is_all_white(line): lineno = lineno + 1 - line = text.get("%d.0" % lineno, "%d.0 lineend" % lineno) + line = text.get("%d.0" % lineno, "%d.end" % lineno) first_lineno = lineno comment_header = get_comment_header(line) comment_header_len = len(comment_header) + + # Once start line found, search for end of paragraph (a blank line) while get_comment_header(line)==comment_header and \ not is_all_white(line[comment_header_len:]): lineno = lineno + 1 - line = text.get("%d.0" % lineno, "%d.0 lineend" % lineno) + line = text.get("%d.0" % lineno, "%d.end" % lineno) last = "%d.0" % lineno - # Search back to beginning of paragraph + + # Search back to beginning of paragraph (first blank line before) lineno = first_lineno - 1 - line = text.get("%d.0" % lineno, "%d.0 lineend" % lineno) + line = text.get("%d.0" % lineno, "%d.end" % lineno) while lineno > 0 and \ get_comment_header(line)==comment_header and \ not is_all_white(line[comment_header_len:]): lineno = lineno - 1 - line = text.get("%d.0" % lineno, "%d.0 lineend" % lineno) + line = text.get("%d.0" % lineno, "%d.end" % lineno) first = "%d.0" % (lineno+1) + return first, last, comment_header, text.get(first, last) +# This should perhaps be replaced with textwrap.wrap def reformat_paragraph(data, limit): + """Return data reformatted to specified width (limit).""" lines = data.split("\n") i = 0 n = len(lines) @@ -126,7 +130,7 @@ def reformat_paragraph(data, limit): if not word: continue # Can happen when line ends in whitespace if len((partial + word).expandtabs()) > limit and \ - partial != indent1: + partial != indent1: new.append(partial.rstrip()) partial = indent2 partial = partial + word + " " @@ -138,13 +142,50 @@ def reformat_paragraph(data, limit): new.extend(lines[i:]) return "\n".join(new) +def reformat_comment(data, limit, comment_header): + """Return data reformatted to specified width with comment header.""" + + # Remove header from the comment lines + lc = len(comment_header) + data = "\n".join(line[lc:] for line in data.split("\n")) + # Reformat to maxformatwidth chars or a 20 char width, + # whichever is greater. + format_width = max(limit - len(comment_header), 20) + newdata = reformat_paragraph(data, format_width) + # re-split and re-insert the comment header. + newdata = newdata.split("\n") + # If the block ends in a \n, we dont want the comment prefix + # inserted after it. (Im not sure it makes sense to reformat a + # comment block that is not made of complete lines, but whatever!) + # Can't think of a clean solution, so we hack away + block_suffix = "" + if not newdata[-1]: + block_suffix = "\n" + newdata = newdata[:-1] + return '\n'.join(comment_header+line for line in newdata) + block_suffix + def is_all_white(line): + """Return True if line is empty or all whitespace.""" + return re.match(r"^\s*$", line) is not None def get_indent(line): - return re.match(r"^(\s*)", line).group() + """Return the initial space or tab indent of line.""" + return re.match(r"^([ \t]*)", line).group() def get_comment_header(line): - m = re.match(r"^(\s*#*)", line) + """Return string with leading whitespace and '#' from line or ''. + + A null return indicates that the line is not a comment line. A non- + null return, such as ' #', will be used to find the other lines of + a comment block with the same indent. + """ + m = re.match(r"^([ \t]*#*)", line) if m is None: return "" return m.group(1) + +if __name__ == "__main__": + from test import support; support.use_resources = ['gui'] + import unittest + unittest.main('idlelib.idle_test.test_formatparagraph', + verbosity=2, exit=False) diff --git a/Lib/idlelib/GrepDialog.py b/Lib/idlelib/GrepDialog.py index e40e5468c0f..b5e35d7454e 100644 --- a/Lib/idlelib/GrepDialog.py +++ b/Lib/idlelib/GrepDialog.py @@ -81,36 +81,24 @@ def grep_it(self, prog, path): hits = 0 for fn in list: try: - f = open(fn) - except IOError, msg: + with open(fn) as f: + for lineno, line in enumerate(f, 1): + if line[-1:] == '\n': + line = line[:-1] + if prog.search(line): + sys.stdout.write("%s: %s: %s\n" % + (fn, lineno, line)) + hits += 1 + except IOError as msg: print msg - continue - lineno = 0 - while 1: - block = f.readlines(100000) - if not block: - break - for line in block: - lineno = lineno + 1 - if line[-1:] == '\n': - line = line[:-1] - if prog.search(line): - sys.stdout.write("%s: %s: %s\n" % (fn, lineno, line)) - hits = hits + 1 - if hits: - if hits == 1: - s = "" - else: - s = "s" - print "Found", hits, "hit%s." % s - print "(Hint: right-click to open locations.)" - else: - print "No hits." + print(("Hits found: %s\n" + "(Hint: right-click to open locations.)" + % hits) if hits else "No hits.") def findfiles(self, dir, base, rec): try: names = os.listdir(dir or os.curdir) - except os.error, msg: + except os.error as msg: print msg return [] list = [] @@ -131,3 +119,9 @@ def close(self, event=None): if self.top: self.top.grab_release() self.top.withdraw() + +if __name__ == "__main__": + # A human test is a bit tricky since EditorWindow() imports this module. + # Hence Idle must be restarted after editing this file for a live test. + import unittest + unittest.main('idlelib.idle_test.test_grep', verbosity=2, exit=False) diff --git a/Lib/idlelib/IOBinding.py b/Lib/idlelib/IOBinding.py index 8078c3f4b76..2eae11b5411 100644 --- a/Lib/idlelib/IOBinding.py +++ b/Lib/idlelib/IOBinding.py @@ -71,7 +71,7 @@ encoding = encoding.lower() -coding_re = re.compile("coding[:=]\s*([-\w_.]+)") +coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') class EncodingMessage(SimpleDialog): "Inform user that an encoding declaration is needed." @@ -125,11 +125,12 @@ def coding_spec(str): Raise LookupError if the encoding is declared but unknown. """ # Only consider the first two lines - str = str.split("\n")[:2] - str = "\n".join(str) - - match = coding_re.search(str) - if not match: + str = str.split("\n", 2)[:2] + for line in lst: + match = coding_re.match(line) + if match is not None: + break + else: return None name = match.group(1) # Check whether the encoding is known @@ -248,10 +249,9 @@ def loadfile(self, filename): try: # open the file in binary mode so that we can handle # end-of-line convention ourselves. - f = open(filename,'rb') - chars = f.read() - f.close() - except IOError, msg: + with open(filename, 'rb') as f: + chars = f.read() + except IOError as msg: tkMessageBox.showerror("I/O Error", str(msg), master=self.text) return False @@ -294,7 +294,7 @@ def decode(self, chars): # Next look for coding specification try: enc = coding_spec(chars) - except LookupError, name: + except LookupError as name: tkMessageBox.showerror( title="Error loading the file", message="The encoding '%s' is not known to this Python "\ @@ -383,12 +383,10 @@ def writefile(self, filename): if self.eol_convention != "\n": chars = chars.replace("\n", self.eol_convention) try: - f = open(filename, "wb") - f.write(chars) - f.flush() - f.close() + with open(filename, "wb") as f: + f.write(chars) return True - except IOError, msg: + except IOError as msg: tkMessageBox.showerror("I/O Error", str(msg), master=self.text) return False @@ -408,7 +406,7 @@ def encode(self, chars): try: enc = coding_spec(chars) failed = None - except LookupError, msg: + except LookupError as msg: failed = msg enc = None if enc: diff --git a/Lib/idlelib/IdleHistory.py b/Lib/idlelib/IdleHistory.py index 983a1406d44..931d5b28e43 100644 --- a/Lib/idlelib/IdleHistory.py +++ b/Lib/idlelib/IdleHistory.py @@ -1,81 +1,93 @@ +"Implement Idle Shell history mechanism with History class" + from idlelib.configHandler import idleConf class History: + ''' Implement Idle Shell history mechanism. + + store - Store source statement (called from PyShell.resetoutput). + fetch - Fetch stored statement matching prefix already entered. + history_next - Bound to <> event (default Alt-N). + history_prev - Bound to <> event (default Alt-P). + ''' + def __init__(self, text): + '''Initialize data attributes and bind event methods. - def __init__(self, text, output_sep = "\n"): + .text - Idle wrapper of tk Text widget, with .bell(). + .history - source statements, possibly with multiple lines. + .prefix - source already entered at prompt; filters history list. + .pointer - index into history. + .cyclic - wrap around history list (or not). + ''' self.text = text self.history = [] - self.history_prefix = None - self.history_pointer = None - self.output_sep = output_sep + self.prefix = None + self.pointer = None self.cyclic = idleConf.GetOption("main", "History", "cyclic", 1, "bool") text.bind("<>", self.history_prev) text.bind("<>", self.history_next) def history_next(self, event): - self.history_do(0) + "Fetch later statement; start with ealiest if cyclic." + self.fetch(reverse=False) return "break" def history_prev(self, event): - self.history_do(1) + "Fetch earlier statement; start with most recent." + self.fetch(reverse=True) return "break" - def _get_source(self, start, end): - # Get source code from start index to end index. Lines in the - # text control may be separated by sys.ps2 . - lines = self.text.get(start, end).split(self.output_sep) - return "\n".join(lines) + def fetch(self, reverse): + '''Fetch statememt and replace current line in text widget. - def _put_source(self, where, source): - output = self.output_sep.join(source.split("\n")) - self.text.insert(where, output) - - def history_do(self, reverse): + Set prefix and pointer as needed for successive fetches. + Reset them to None, None when returning to the start line. + Sound bell when return to start line or cannot leave a line + because cyclic is False. + ''' nhist = len(self.history) - pointer = self.history_pointer - prefix = self.history_prefix + pointer = self.pointer + prefix = self.prefix if pointer is not None and prefix is not None: if self.text.compare("insert", "!=", "end-1c") or \ - self._get_source("iomark", "end-1c") != self.history[pointer]: + self.text.get("iomark", "end-1c") != self.history[pointer]: pointer = prefix = None + self.text.mark_set("insert", "end-1c") # != after cursor move if pointer is None or prefix is None: - prefix = self._get_source("iomark", "end-1c") + prefix = self.text.get("iomark", "end-1c") if reverse: - pointer = nhist + pointer = nhist # will be decremented else: if self.cyclic: - pointer = -1 - else: + pointer = -1 # will be incremented + else: # abort history_next self.text.bell() return nprefix = len(prefix) while 1: - if reverse: - pointer = pointer - 1 - else: - pointer = pointer + 1 + pointer += -1 if reverse else 1 if pointer < 0 or pointer >= nhist: self.text.bell() - if not self.cyclic and pointer < 0: + if not self.cyclic and pointer < 0: # abort history_prev return else: - if self._get_source("iomark", "end-1c") != prefix: + if self.text.get("iomark", "end-1c") != prefix: self.text.delete("iomark", "end-1c") - self._put_source("iomark", prefix) + self.text.insert("iomark", prefix) pointer = prefix = None break item = self.history[pointer] if item[:nprefix] == prefix and len(item) > nprefix: self.text.delete("iomark", "end-1c") - self._put_source("iomark", item) + self.text.insert("iomark", item) break - self.text.mark_set("insert", "end-1c") self.text.see("insert") self.text.tag_remove("sel", "1.0", "end") - self.history_pointer = pointer - self.history_prefix = prefix + self.pointer = pointer + self.prefix = prefix - def history_store(self, source): + def store(self, source): + "Store Shell input statement into history list." source = source.strip() if len(source) > 2: # avoid duplicates @@ -84,5 +96,11 @@ def history_store(self, source): except ValueError: pass self.history.append(source) - self.history_pointer = None - self.history_prefix = None + self.pointer = None + self.prefix = None + +if __name__ == "__main__": + from test import test_support as support + support.use_resources = ['gui'] + from unittest import main + main('idlelib.idle_test.test_idlehistory', verbosity=2, exit=False) diff --git a/Lib/idlelib/PathBrowser.py b/Lib/idlelib/PathBrowser.py index d88a48e344e..0f4fb3e8ba0 100644 --- a/Lib/idlelib/PathBrowser.py +++ b/Lib/idlelib/PathBrowser.py @@ -92,4 +92,5 @@ def main(): mainloop() if __name__ == "__main__": - main() + from unittest import main + main('idlelib.idle_test.test_pathbrowser', verbosity=2, exit=False) diff --git a/Lib/idlelib/PyShell.py b/Lib/idlelib/PyShell.py index d144a51b725..7a9a0bc01b9 100644 --- a/Lib/idlelib/PyShell.py +++ b/Lib/idlelib/PyShell.py @@ -50,35 +50,55 @@ # internal warnings to the console. ScriptBinding.check_syntax() will # temporarily redirect the stream to the shell window to display warnings when # checking user's code. -global warning_stream -warning_stream = sys.__stderr__ -try: - import warnings -except ImportError: - pass -else: - def idle_showwarning(message, category, filename, lineno, - file=None, line=None): - if file is None: - file = warning_stream - try: - file.write(warnings.formatwarning(message, category, filename, - lineno, line=line)) - except IOError: - pass ## file (probably __stderr__) is invalid, warning dropped. - warnings.showwarning = idle_showwarning - def idle_formatwarning(message, category, filename, lineno, line=None): - """Format warnings the IDLE way""" - s = "\nWarning (from warnings module):\n" - s += ' File \"%s\", line %s\n' % (filename, lineno) - if line is None: - line = linecache.getline(filename, lineno) - line = line.strip() - if line: - s += " %s\n" % line - s += "%s: %s\n>>> " % (category.__name__, message) - return s - warnings.formatwarning = idle_formatwarning +warning_stream = sys.__stderr__ # None, at least on Windows, if no console. +import warnings + +def idle_formatwarning(message, category, filename, lineno, line=None): + """Format warnings the IDLE way.""" + + s = "\nWarning (from warnings module):\n" + s += ' File \"%s\", line %s\n' % (filename, lineno) + if line is None: + line = linecache.getline(filename, lineno) + line = line.strip() + if line: + s += " %s\n" % line + s += "%s: %s\n" % (category.__name__, message) + return s + +def idle_showwarning( + message, category, filename, lineno, file=None, line=None): + """Show Idle-format warning (after replacing warnings.showwarning). + + The differences are the formatter called, the file=None replacement, + which can be None, the capture of the consequence AttributeError, + and the output of a hard-coded prompt. + """ + if file is None: + file = warning_stream + try: + file.write(idle_formatwarning( + message, category, filename, lineno, line=line)) + file.write(">>> ") + except (AttributeError, IOError): + pass # if file (probably __stderr__) is invalid, skip warning. + +_warnings_showwarning = None + +def capture_warnings(capture): + "Replace warning.showwarning with idle_showwarning, or reverse." + + global _warnings_showwarning + if capture: + if _warnings_showwarning is None: + _warnings_showwarning = warnings.showwarning + warnings.showwarning = idle_showwarning + else: + if _warnings_showwarning is not None: + warnings.showwarning = _warnings_showwarning + _warnings_showwarning = None + +capture_warnings(True) def extended_linecache_checkcache(filename=None, orig_checkcache=linecache.checkcache): @@ -370,6 +390,7 @@ def __init__(self, tkconsole): self.port = PORT self.original_compiler_flags = self.compile.compiler.flags + _afterid = None rpcclt = None rpcpid = None @@ -409,7 +430,7 @@ def start_subprocess(self): try: self.rpcclt = MyRPCClient(addr) break - except socket.error, err: + except socket.error as err: pass else: self.display_port_binding_error() @@ -430,7 +451,7 @@ def start_subprocess(self): self.rpcclt.listening_sock.settimeout(10) try: self.rpcclt.accept() - except socket.timeout, err: + except socket.timeout as err: self.display_no_subprocess_error() return None self.rpcclt.register("console", self.tkconsole) @@ -465,10 +486,11 @@ def restart_subprocess(self, with_cwd=False): self.spawn_subprocess() try: self.rpcclt.accept() - except socket.timeout, err: + except socket.timeout as err: self.display_no_subprocess_error() return None self.transfer_path(with_cwd=with_cwd) + console.stop_readline() # annotate restart in shell window and mark it console.text.delete("iomark", "end-1c") if was_executing: @@ -496,6 +518,8 @@ def interrupt_subprocess(self): threading.Thread(target=self.__request_interrupt).start() def kill_subprocess(self): + if self._afterid is not None: + self.tkconsole.text.after_cancel(self._afterid) try: self.rpcclt.close() except AttributeError: # no socket @@ -568,8 +592,8 @@ def poll_subprocess(self): pass # Reschedule myself if not self.tkconsole.closing: - self.tkconsole.text.after(self.tkconsole.pollinterval, - self.poll_subprocess) + self._afterid = self.tkconsole.text.after( + self.tkconsole.pollinterval, self.poll_subprocess) debugger = None @@ -843,7 +867,6 @@ class PyShell(OutputWindow): ] if macosxSupport.runningAsOSXApp(): - del menu_specs[-3] menu_specs[-2] = ("windows", "_Window") @@ -908,6 +931,7 @@ def get_standard_extension_names(self): canceled = False endoffile = False closing = False + _stop_readline_flag = False def set_warning_stream(self, stream): global warning_stream @@ -983,14 +1007,9 @@ def close(self): parent=self.text) if response is False: return "cancel" - if self.reading: - self.top.quit() + self.stop_readline() self.canceled = True self.closing = True - # Wait for poll_subprocess() rescheduling to stop - self.text.after(2 * self.pollinterval, self.close2) - - def close2(self): return EditorWindow.close(self) def _close(self): @@ -1036,6 +1055,12 @@ def begin(self): Tkinter._default_root = None # 03Jan04 KBK What's this? return True + def stop_readline(self): + if not self.reading: # no nested mainloop to exit. + return + self._stop_readline_flag = True + self.top.quit() + def readline(self): save = self.reading try: @@ -1043,6 +1068,9 @@ def readline(self): self.top.mainloop() # nested mainloop() finally: self.reading = save + if self._stop_readline_flag: + self._stop_readline_flag = False + return "" line = self.text.get("iomark", "end-1c") if len(line) == 0: # may be EOF if we quit our mainloop with Ctrl-C line = "\n" @@ -1250,7 +1278,7 @@ def showprompt(self): def resetoutput(self): source = self.text.get("iomark", "end-1c") if self.history: - self.history.history_store(source) + self.history.store(source) if self.text.get("end-2c") != "\n": self.text.insert("end-1c", "\n") self.text.mark_set("iomark", "end-1c") @@ -1360,6 +1388,9 @@ def readline(self, size=-1): self._line_buffer = line[size:] return line[:size] + def close(self): + self.shell.close() + usage_msg = """\ @@ -1417,6 +1448,7 @@ def readline(self, size=-1): def main(): global flist, root, use_subprocess + capture_warnings(True) use_subprocess = True enable_shell = False enable_edit = False @@ -1426,7 +1458,7 @@ def main(): startup = False try: opts, args = getopt.getopt(sys.argv[1:], "c:deihnr:st:") - except getopt.error, msg: + except getopt.error as msg: sys.stderr.write("Error: %s\n" % str(msg)) sys.stderr.write(usage_msg) sys.exit(2) @@ -1549,7 +1581,10 @@ def main(): while flist.inversedict: # keep IDLE running while files are open. root.mainloop() root.destroy() + capture_warnings(False) if __name__ == "__main__": sys.modules['PyShell'] = sys.modules['__main__'] main() + +capture_warnings(False) # Make sure turned off; see issue 18081 diff --git a/Lib/idlelib/ReplaceDialog.py b/Lib/idlelib/ReplaceDialog.py index ae440353f7b..54c270df23c 100644 --- a/Lib/idlelib/ReplaceDialog.py +++ b/Lib/idlelib/ReplaceDialog.py @@ -122,6 +122,7 @@ def replace_all(self, event=None): text.undo_block_stop() if first and last: self.show_hit(first, last) + self.close() def do_find(self, ok=0): if not self.engine.getprog(): diff --git a/Lib/idlelib/RstripExtension.py b/Lib/idlelib/RstripExtension.py index 19e35d4d481..2ce3c7eafe5 100644 --- a/Lib/idlelib/RstripExtension.py +++ b/Lib/idlelib/RstripExtension.py @@ -1,13 +1,9 @@ 'Provides "Strip trailing whitespace" under the "Format" menu.' -__author__ = "Roger D. Serwy " - class RstripExtension: menudefs = [ - ('format', [None, - ('Strip trailing whitespace', '<>'), - ]),] + ('format', [None, ('Strip trailing whitespace', '<>'), ] ), ] def __init__(self, editwin): self.editwin = editwin @@ -20,10 +16,18 @@ def do_rstrip(self, event=None): undo.undo_block_start() - end_line = int(float(text.index('end'))) + 1 + end_line = int(float(text.index('end'))) for cur in range(1, end_line): - txt = text.get('%i.0' % cur, '%i.0 lineend' % cur) + txt = text.get('%i.0' % cur, '%i.end' % cur) + raw = len(txt) cut = len(txt.rstrip()) - text.delete('%i.%i' % (cur, cut), '%i.0 lineend' % cur) + # Since text.delete() marks file as changed, even if not, + # only call it when needed to actually delete something. + if cut < raw: + text.delete('%i.%i' % (cur, cut), '%i.end' % cur) undo.undo_block_stop() + +if __name__ == "__main__": + import unittest + unittest.main('idlelib.idle_test.test_rstrip', verbosity=2, exit=False) diff --git a/Lib/idlelib/ScriptBinding.py b/Lib/idlelib/ScriptBinding.py index 01ac4746589..a024ebfb58e 100644 --- a/Lib/idlelib/ScriptBinding.py +++ b/Lib/idlelib/ScriptBinding.py @@ -70,13 +70,13 @@ def tabnanny(self, filename): f = open(filename, 'r') try: tabnanny.process_tokens(tokenize.generate_tokens(f.readline)) - except tokenize.TokenError, msg: + except tokenize.TokenError as msg: msgtxt, (lineno, start) = msg self.editwin.gotoline(lineno) self.errorbox("Tabnanny Tokenizing Error", "Token Error: %s" % msgtxt) return False - except tabnanny.NannyNag, nag: + except tabnanny.NannyNag as nag: # The error messages from tabnanny are too confusing... self.editwin.gotoline(nag.get_lineno()) self.errorbox("Tab/space error", indent_message) @@ -87,9 +87,8 @@ def checksyntax(self, filename): self.shell = shell = self.flist.open_shell() saved_stream = shell.get_warning_stream() shell.set_warning_stream(shell.stderr) - f = open(filename, 'r') - source = f.read() - f.close() + with open(filename, 'r') as f: + source = f.read() if '\r' in source: source = re.sub(r"\r\n", "\n", source) source = re.sub(r"\r", "\n", source) @@ -101,7 +100,7 @@ def checksyntax(self, filename): try: # If successful, return the compiled code return compile(source, filename, "exec") - except (SyntaxError, OverflowError, ValueError), err: + except (SyntaxError, OverflowError, ValueError) as err: try: msg, (errorfilename, lineno, offset, line) = err if not errorfilename: @@ -152,16 +151,16 @@ def run_module_event(self, event): dirname = os.path.dirname(filename) # XXX Too often this discards arguments the user just set... interp.runcommand("""if 1: - _filename = %r + __file__ = {filename!r} import sys as _sys from os.path import basename as _basename if (not _sys.argv or - _basename(_sys.argv[0]) != _basename(_filename)): - _sys.argv = [_filename] + _basename(_sys.argv[0]) != _basename(__file__)): + _sys.argv = [__file__] import os as _os - _os.chdir(%r) - del _filename, _sys, _basename, _os - \n""" % (filename, dirname)) + _os.chdir({dirname!r}) + del _sys, _basename, _os + \n""".format(filename=filename, dirname=dirname)) interp.prepend_syspath(filename) # XXX KBK 03Jul04 When run w/o subprocess, runtime warnings still # go to __stderr__. With subprocess, they go to the shell. diff --git a/Lib/idlelib/SearchDialog.py b/Lib/idlelib/SearchDialog.py index 7c70b84ee46..3ba0cdb95d1 100644 --- a/Lib/idlelib/SearchDialog.py +++ b/Lib/idlelib/SearchDialog.py @@ -24,13 +24,12 @@ class SearchDialog(SearchDialogBase): def create_widgets(self): f = SearchDialogBase.create_widgets(self) - self.make_button("Find", self.default_command, 1) + self.make_button("Find Next", self.default_command, 1) def default_command(self, event=None): if not self.engine.getprog(): return - if self.find_again(self.text): - self.close() + self.find_again(self.text) def find_again(self, text): if not self.engine.getpat(): diff --git a/Lib/idlelib/SearchDialogBase.py b/Lib/idlelib/SearchDialogBase.py index f63e7ae37c7..13ee90dfa4b 100644 --- a/Lib/idlelib/SearchDialogBase.py +++ b/Lib/idlelib/SearchDialogBase.py @@ -1,6 +1,23 @@ +'''Define SearchDialogBase used by Search, Replace, and Grep dialogs.''' from Tkinter import * class SearchDialogBase: + '''Create most of a modal search dialog (make_frame, create_widgets). + + The wide left column contains: + 1 or 2 text entry lines (create_entries, make_entry); + a row of standard radiobuttons (create_option_buttons); + a row of dialog specific radiobuttons (create_other_buttons). + + The narrow right column contains command buttons + (create_command_buttons, make_button). + These are bound to functions that execute the command. + + Except for command buttons, this base class is not limited to + items common to all three subclasses. Rather, it is the Find dialog + minus the "Find Next" command and its execution function. + The other dialogs override methods to replace and add widgets. + ''' title = "Search Dialog" icon = "Search" diff --git a/Lib/idlelib/SearchEngine.py b/Lib/idlelib/SearchEngine.py index cc40a00c50f..ea6f1dede69 100644 --- a/Lib/idlelib/SearchEngine.py +++ b/Lib/idlelib/SearchEngine.py @@ -1,26 +1,34 @@ +'''Define SearchEngine for search dialogs.''' import re -from Tkinter import * +from Tkinter import StringVar, BooleanVar, TclError import tkMessageBox def get(root): + '''Return the singleton SearchEngine instance for the process. + + The single SearchEngine saves settings between dialog instances. + If there is not a SearchEngine already, make one. + ''' if not hasattr(root, "_searchengine"): root._searchengine = SearchEngine(root) - # XXX This will never garbage-collect -- who cares + # This creates a cycle that persists until root is deleted. return root._searchengine class SearchEngine: + """Handles searching a text widget for Find, Replace, and Grep.""" def __init__(self, root): - self.root = root - # State shared by search, replace, and grep; - # the search dialogs bind these to UI elements. - self.patvar = StringVar(root) # search pattern - self.revar = BooleanVar(root) # regular expression? - self.casevar = BooleanVar(root) # match case? - self.wordvar = BooleanVar(root) # match whole word? - self.wrapvar = BooleanVar(root) # wrap around buffer? - self.wrapvar.set(1) # (on by default) - self.backvar = BooleanVar(root) # search backwards? + '''Initialize Variables that save search state. + + The dialogs bind these to the UI elements present in the dialogs. + ''' + self.root = root # need for report_error() + self.patvar = StringVar(root, '') # search pattern + self.revar = BooleanVar(root, False) # regular expression? + self.casevar = BooleanVar(root, False) # match case? + self.wordvar = BooleanVar(root, False) # match whole word? + self.wrapvar = BooleanVar(root, True) # wrap around buffer? + self.backvar = BooleanVar(root, False) # search backwards? # Access methods @@ -47,15 +55,23 @@ def isback(self): # Higher level access methods + def setcookedpat(self, pat): + "Set pattern after escaping if re." + # called only in SearchDialog.py: 66 + if self.isre(): + pat = re.escape(pat) + self.setpat(pat) + def getcookedpat(self): pat = self.getpat() - if not self.isre(): + if not self.isre(): # if True, see setcookedpat pat = re.escape(pat) if self.isword(): pat = r"\b%s\b" % pat return pat def getprog(self): + "Return compiled cooked search pattern." pat = self.getpat() if not pat: self.report_error(pat, "Empty regular expression") @@ -66,7 +82,7 @@ def getprog(self): flags = flags | re.IGNORECASE try: prog = re.compile(pat, flags) - except re.error, what: + except re.error as what: try: msg, col = what except: @@ -77,40 +93,33 @@ def getprog(self): return prog def report_error(self, pat, msg, col=-1): - # Derived class could overrid this with something fancier + # Derived class could override this with something fancier msg = "Error: " + str(msg) if pat: - msg = msg + "\np\Pattern: " + str(pat) + msg = msg + "\nPattern: " + str(pat) if col >= 0: msg = msg + "\nOffset: " + str(col) tkMessageBox.showerror("Regular expression error", msg, master=self.root) - def setcookedpat(self, pat): - if self.isre(): - pat = re.escape(pat) - self.setpat(pat) - def search_text(self, text, prog=None, ok=0): - """Search a text widget for the pattern. + '''Return (lineno, matchobj) or None for forward/backward search. - If prog is given, it should be the precompiled pattern. - Return a tuple (lineno, matchobj); None if not found. + This function calls the right function with the right arguments. + It directly return the result of that call. - This obeys the wrap and direction (back) settings. + Text is a text widget. Prog is a precompiled pattern. + The ok parameteris a bit complicated as it has two effects. - The search starts at the selection (if there is one) or - at the insert mark (otherwise). If the search is forward, - it starts at the right of the selection; for a backward - search, it starts at the left end. An empty match exactly - at either end of the selection (or at the insert mark if - there is no selection) is ignored unless the ok flag is true - -- this is done to guarantee progress. + If there is a selection, the search begin at either end, + depending on the direction setting and ok, with ok meaning that + the search starts with the selection. Otherwise, search begins + at the insert mark. - If the search is allowed to wrap around, it will return the - original selection if (and only if) it is the only match. + To aid progress, the search functions do not return an empty + match at the starting position unless ok is True. + ''' - """ if not prog: prog = self.getprog() if not prog: @@ -179,15 +188,19 @@ def search_backward(self, text, prog, line, col, wrap, ok=0): col = len(chars) - 1 return None -# Helper to search backwards in a string. -# (Optimized for the case where the pattern isn't found.) - def search_reverse(prog, chars, col): + '''Search backwards and return an re match object or None. + + This is done by searching forwards until there is no match. + Prog: compiled re object with a search method returning a match. + Chars: line of text, without \n. + Col: stop index for the search; the limit for match.end(). + ''' m = prog.search(chars) if not m: return None found = None - i, j = m.span() + i, j = m.span() # m.start(), m.end() == match slice indexes while i < col and j <= col: found = m if i == j: @@ -198,10 +211,9 @@ def search_reverse(prog, chars, col): i, j = m.span() return found -# Helper to get selection end points, defaulting to insert mark. -# Return a tuple of indices ("line.col" strings). - def get_selection(text): + '''Return tuple of 'line.col' indexes from selection or insert mark. + ''' try: first = text.index("sel.first") last = text.index("sel.last") @@ -213,8 +225,12 @@ def get_selection(text): last = first return first, last -# Helper to parse a text index into a (line, col) tuple. - def get_line_col(index): + '''Return (line, col) tuple of ints from 'line.col' string.''' line, col = map(int, index.split(".")) # Fails on invalid index return line, col + +if __name__ == "__main__": + from test import support; support.use_resources = ['gui'] + import unittest + unittest.main('idlelib.idle_test.test_searchengine', verbosity=2, exit=False) diff --git a/Lib/idlelib/aboutDialog.py b/Lib/idlelib/aboutDialog.py index 43a13135aee..07345180f91 100644 --- a/Lib/idlelib/aboutDialog.py +++ b/Lib/idlelib/aboutDialog.py @@ -66,12 +66,7 @@ def CreateWidgets(self): labelPythonVer = Label(frameBg, text='Python version: ' + \ sys.version.split()[0], fg=self.fg, bg=self.bg) labelPythonVer.grid(row=9, column=0, sticky=W, padx=10, pady=0) - # handle weird tk version num in windoze python >= 1.6 (?!?) - tkVer = repr(TkVersion).split('.') - tkVer[len(tkVer)-1] = str('%.3g' % (float('.'+tkVer[len(tkVer)-1])))[2:] - if tkVer[len(tkVer)-1] == '': - tkVer[len(tkVer)-1] = '0' - tkVer = '.'.join(tkVer) + tkVer = self.tk.call('info', 'patchlevel') labelTkVer = Label(frameBg, text='Tk version: '+ tkVer, fg=self.fg, bg=self.bg) labelTkVer.grid(row=9, column=1, sticky=W, padx=2, pady=0) diff --git a/Lib/idlelib/configSectionNameDialog.py b/Lib/idlelib/configSectionNameDialog.py index 4f1b002afca..04fcd8cf9a9 100644 --- a/Lib/idlelib/configSectionNameDialog.py +++ b/Lib/idlelib/configSectionNameDialog.py @@ -1,97 +1,100 @@ """ Dialog that allows user to specify a new config file section name. Used to get new highlight theme and keybinding set names. +The 'return value' for the dialog, used two placed in configDialog.py, +is the .result attribute set in the Ok and Cancel methods. """ from Tkinter import * import tkMessageBox - class GetCfgSectionNameDialog(Toplevel): - def __init__(self,parent,title,message,usedNames): + def __init__(self, parent, title, message, used_names): """ message - string, informational message to display - usedNames - list, list of names already in use for validity check + used_names - string collection, names already in use for validity check """ Toplevel.__init__(self, parent) self.configure(borderwidth=5) - self.resizable(height=FALSE,width=FALSE) + self.resizable(height=FALSE, width=FALSE) self.title(title) self.transient(parent) self.grab_set() self.protocol("WM_DELETE_WINDOW", self.Cancel) self.parent = parent - self.message=message - self.usedNames=usedNames - self.result='' - self.CreateWidgets() - self.withdraw() #hide while setting geometry + self.message = message + self.used_names = used_names + self.create_widgets() + self.withdraw() #hide while setting geometry self.update_idletasks() #needs to be done here so that the winfo_reqwidth is valid self.messageInfo.config(width=self.frameMain.winfo_reqwidth()) - self.geometry("+%d+%d" % - ((parent.winfo_rootx()+((parent.winfo_width()/2) - -(self.winfo_reqwidth()/2)), - parent.winfo_rooty()+((parent.winfo_height()/2) - -(self.winfo_reqheight()/2)) )) ) #centre dialog over parent - self.deiconify() #geometry set, unhide + self.geometry( + "+%d+%d" % ( + parent.winfo_rootx() + + (parent.winfo_width()/2 - self.winfo_reqwidth()/2), + parent.winfo_rooty() + + (parent.winfo_height()/2 - self.winfo_reqheight()/2) + ) ) #centre dialog over parent + self.deiconify() #geometry set, unhide self.wait_window() - - def CreateWidgets(self): - self.name=StringVar(self) - self.fontSize=StringVar(self) - self.frameMain = Frame(self,borderwidth=2,relief=SUNKEN) - self.frameMain.pack(side=TOP,expand=TRUE,fill=BOTH) - self.messageInfo=Message(self.frameMain,anchor=W,justify=LEFT,padx=5,pady=5, - text=self.message)#,aspect=200) - entryName=Entry(self.frameMain,textvariable=self.name,width=30) + def create_widgets(self): + self.name = StringVar(self.parent) + self.fontSize = StringVar(self.parent) + self.frameMain = Frame(self, borderwidth=2, relief=SUNKEN) + self.frameMain.pack(side=TOP, expand=TRUE, fill=BOTH) + self.messageInfo = Message(self.frameMain, anchor=W, justify=LEFT, + padx=5, pady=5, text=self.message) #,aspect=200) + entryName = Entry(self.frameMain, textvariable=self.name, width=30) entryName.focus_set() - self.messageInfo.pack(padx=5,pady=5)#,expand=TRUE,fill=BOTH) - entryName.pack(padx=5,pady=5) - frameButtons=Frame(self) - frameButtons.pack(side=BOTTOM,fill=X) - self.buttonOk = Button(frameButtons,text='Ok', - width=8,command=self.Ok) - self.buttonOk.grid(row=0,column=0,padx=5,pady=5) - self.buttonCancel = Button(frameButtons,text='Cancel', - width=8,command=self.Cancel) - self.buttonCancel.grid(row=0,column=1,padx=5,pady=5) + self.messageInfo.pack(padx=5, pady=5) #, expand=TRUE, fill=BOTH) + entryName.pack(padx=5, pady=5) + frameButtons = Frame(self, pady=2) + frameButtons.pack(side=BOTTOM) + self.buttonOk = Button(frameButtons, text='Ok', + width=8, command=self.Ok) + self.buttonOk.pack(side=LEFT, padx=5) + self.buttonCancel = Button(frameButtons, text='Cancel', + width=8, command=self.Cancel) + self.buttonCancel.pack(side=RIGHT, padx=5) - def NameOk(self): - #simple validity check for a sensible - #ConfigParser file section name - nameOk=1 - name=self.name.get() - name.strip() + def name_ok(self): + ''' After stripping entered name, check that it is a sensible + ConfigParser file section name. Return it if it is, '' if not. + ''' + name = self.name.get().strip() if not name: #no name specified tkMessageBox.showerror(title='Name Error', message='No name specified.', parent=self) - nameOk=0 elif len(name)>30: #name too long tkMessageBox.showerror(title='Name Error', message='Name too long. It should be no more than '+ '30 characters.', parent=self) - nameOk=0 - elif name in self.usedNames: + name = '' + elif name in self.used_names: tkMessageBox.showerror(title='Name Error', message='This name is already in use.', parent=self) - nameOk=0 - return nameOk - + name = '' + return name def Ok(self, event=None): - if self.NameOk(): - self.result=self.name.get().strip() + name = self.name_ok() + if name: + self.result = name self.destroy() - def Cancel(self, event=None): - self.result='' + self.result = '' self.destroy() - if __name__ == '__main__': - #test the dialog - root=Tk() + import unittest + unittest.main('idlelib.idle_test.test_config_name', verbosity=2, exit=False) + + # also human test the dialog + root = Tk() def run(): - keySeq='' dlg=GetCfgSectionNameDialog(root,'Get Name', - 'The information here should need to be word wrapped. Test.') + "After the text entered with [Ok] is stripped, , " + "'abc', or more that 30 chars are errors. " + "Close with a valid entry (printed), [Cancel], or [X]", + {'abc'}) print dlg.result - Button(root,text='Dialog',command=run).pack() + Message(root, text='').pack() # will be needed for oher dialog tests + Button(root, text='Click to begin dialog test', command=run).pack() root.mainloop() diff --git a/Lib/idlelib/help.txt b/Lib/idlelib/help.txt index 2ebf8bf02a5..bd6822c4233 100644 --- a/Lib/idlelib/help.txt +++ b/Lib/idlelib/help.txt @@ -5,7 +5,7 @@ separate window containing the menu is created. File Menu: - New Window -- Create a new editing window + New File -- Create a new editing window Open... -- Open an existing file Recent Files... -- Open a list of recent files Open Module... -- Open an existing module (searches sys.path) @@ -233,8 +233,7 @@ Completions: Python Shell window: Control-c interrupts executing command. - Control-d sends end-of-file; closes window if typed at >>> prompt - (this is Control-z on Windows). + Control-d sends end-of-file; closes window if typed at >>> prompt. Command history: diff --git a/Lib/idlelib/idle_test/README.txt b/Lib/idlelib/idle_test/README.txt new file mode 100644 index 00000000000..a8d4dcb9922 --- /dev/null +++ b/Lib/idlelib/idle_test/README.txt @@ -0,0 +1,108 @@ +README FOR IDLE TESTS IN IDLELIB.IDLE_TEST + + +1. Test Files + +The idle directory, idlelib, has over 60 xyz.py files. The idle_test +subdirectory should contain a test_xyy.py for each. (For test modules, make +'xyz' lower case, and possibly shorten it.) Each file should start with the +something like the following template, with the blanks after after '.' and 'as', +and before and after '_' filled in. +--- +import unittest +from test.support import requires +import idlelib. as + +class _Test(unittest.TestCase): + + def test_(self): + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=2) +--- +Idle tests are run with unittest; do not use regrtest's test_main. + +Once test_xyy is written, the following should go at the end of xyy.py, +with xyz (lowercased) added after 'test_'. +--- +if __name__ == "__main__": + from test import support; support.use_resources = ['gui'] + import unittest + unittest.main('idlelib.idle_test.test_', verbosity=2, exit=False) +--- + + +2. Gui Tests + +Gui tests need 'requires' and 'use_resources' from test.support +(test.test_support in 2.7). A test is a gui test if it creates a Tk root or +master object either directly or indirectly by instantiating a tkinter or +idle class. For the benefit of buildbot machines that do not have a graphics +screen, gui tests must be 'guarded' by "requires('gui')" in a setUp +function or method. This will typically be setUpClass. + +All gui objects must be destroyed by the end of the test, perhaps in a tearDown +function. Creating the Tk root directly in a setUp allows a reference to be saved +so it can be properly destroyed in the corresponding tearDown. +--- + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = tk.Tk() + + @classmethod + def tearDownClass(cls): + cls.root.destroy() +--- + +Support.requires('gui') returns true if it is either called in a main module +(which never happens on buildbots) or if use_resources contains 'gui'. +Use_resources is set by test.regrtest but not by unittest. So when running +tests in another module with unittest, we set it ourselves, as in the xyz.py +template above. + +Since non-gui tests always run, but gui tests only sometimes, tests of non-gui +operations should best avoid needing a gui. Methods that make incidental use of +tkinter (tk) variables and messageboxes can do this by using the mock classes in +idle_test/mock_tk.py. There is also a mock text that will handle some uses of the +tk Text widget. + + +3. Running Tests + +Assume that xyz.py and test_xyz.py end with the "if __name__" statements given +above. In Idle, pressing F5 in an editor window with either loaded will run all +tests in the test_xyz file with the version of Python running Idle. The test +report and any tracebacks will appear in the Shell window. The options in these +"if __name__" statements are appropriate for developers running (as opposed to +importing) either of the files during development: verbosity=2 lists all test +methods in the file; exit=False avoids a spurious sys.exit traceback that would +otherwise occur when running in Idle. The following command lines also run +all test methods, including gui tests, in test_xyz.py. (The exceptions are that +idlelib and idlelib.idle start Idle and idlelib.PyShell should (issue 18330).) + +python -m idlelib.xyz # With the capitalization of the xyz module +python -m idlelib.idle_test.test_xyz + +To run all idle_test/test_*.py tests, either interactively +('>>>', with unittest imported) or from a command line, use one of the +following. (Notes: unittest does not run gui tests; in 2.7, 'test ' (with the +space) is 'test.regrtest '; where present, -v and -ugui can be omitted.) + +>>> unittest.main('idlelib.idle_test', verbosity=2, exit=False) +python -m unittest -v idlelib.idle_test +python -m test -v -ugui test_idle +python -m test.test_idle + +The idle tests are 'discovered' by idlelib.idle_test.__init__.load_tests, +which is also imported into test.test_idle. Normally, neither file should be +changed when working on individual test modules. The third command runs runs +unittest indirectly through regrtest. The same happens when the entire test +suite is run with 'python -m test'. So that command must work for buildbots +to stay green. Idle tests must not disturb the environment in a way that +makes other tests fail (issue 18081). + +To run an individual Testcase or test method, extend the dotted name given to +unittest on the command line. (But gui tests will not this way.) + +python -m unittest -v idlelib.idle_test.text_xyz.Test_case.test_meth diff --git a/Lib/idlelib/idle_test/__init__.py b/Lib/idlelib/idle_test/__init__.py new file mode 100644 index 00000000000..1bc953643e1 --- /dev/null +++ b/Lib/idlelib/idle_test/__init__.py @@ -0,0 +1,9 @@ +from os.path import dirname + +def load_tests(loader, standard_tests, pattern): + this_dir = dirname(__file__) + top_dir = dirname(dirname(this_dir)) + package_tests = loader.discover(start_dir=this_dir, pattern='test*.py', + top_level_dir=top_dir) + standard_tests.addTests(package_tests) + return standard_tests diff --git a/Lib/idlelib/idle_test/mock_idle.py b/Lib/idlelib/idle_test/mock_idle.py new file mode 100644 index 00000000000..54834de2740 --- /dev/null +++ b/Lib/idlelib/idle_test/mock_idle.py @@ -0,0 +1,27 @@ +'''Mock classes that imitate idlelib modules or classes. + +Attributes and methods will be added as needed for tests. +''' + +from idlelib.idle_test.mock_tk import Text + +class Editor(object): + '''Minimally imitate EditorWindow.EditorWindow class. + ''' + def __init__(self, flist=None, filename=None, key=None, root=None): + self.text = Text() + self.undo = UndoDelegator() + + def get_selection_indices(self): + first = self.text.index('1.0') + last = self.text.index('end') + return first, last + +class UndoDelegator(object): + '''Minimally imitate UndoDelegator,UndoDelegator class. + ''' + # A real undo block is only needed for user interaction. + def undo_block_start(*args): + pass + def undo_block_stop(*args): + pass diff --git a/Lib/idlelib/idle_test/mock_tk.py b/Lib/idlelib/idle_test/mock_tk.py new file mode 100644 index 00000000000..9ce70905ed2 --- /dev/null +++ b/Lib/idlelib/idle_test/mock_tk.py @@ -0,0 +1,279 @@ +"""Classes that replace tkinter gui objects used by an object being tested. + +A gui object is anything with a master or parent paramenter, which is typically +required in spite of what the doc strings say. +""" + +class Var(object): + "Use for String/Int/BooleanVar: incomplete" + def __init__(self, master=None, value=None, name=None): + self.master = master + self.value = value + self.name = name + def set(self, value): + self.value = value + def get(self): + return self.value + +class Mbox_func(object): + """Generic mock for messagebox functions, which all have the same signature. + + Instead of displaying a message box, the mock's call method saves the + arguments as instance attributes, which test functions can then examime. + """ + def __init__(self): + self.result = None # The return for all show funcs + def __call__(self, title, message, *args, **kwds): + # Save all args for possible examination by tester + self.title = title + self.message = message + self.args = args + self.kwds = kwds + return self.result # Set by tester for ask functions + +class Mbox(object): + """Mock for tkinter.messagebox with an Mbox_func for each function. + + This module was 'tkMessageBox' in 2.x; hence the 'import as' in 3.x. + Example usage in test_module.py for testing functions in module.py: + --- +from idlelib.idle_test.mock_tk import Mbox +import module + +orig_mbox = module.tkMessageBox +showerror = Mbox.showerror # example, for attribute access in test methods + +class Test(unittest.TestCase): + + @classmethod + def setUpClass(cls): + module.tkMessageBox = Mbox + + @classmethod + def tearDownClass(cls): + module.tkMessageBox = orig_mbox + --- + For 'ask' functions, set func.result return value before calling the method + that uses the message function. When tkMessageBox functions are the + only gui alls in a method, this replacement makes the method gui-free, + """ + askokcancel = Mbox_func() # True or False + askquestion = Mbox_func() # 'yes' or 'no' + askretrycancel = Mbox_func() # True or False + askyesno = Mbox_func() # True or False + askyesnocancel = Mbox_func() # True, False, or None + showerror = Mbox_func() # None + showinfo = Mbox_func() # None + showwarning = Mbox_func() # None + +from _tkinter import TclError + +class Text(object): + """A semi-functional non-gui replacement for tkinter.Text text editors. + + The mock's data model is that a text is a list of \n-terminated lines. + The mock adds an empty string at the beginning of the list so that the + index of actual lines start at 1, as with Tk. The methods never see this. + Tk initializes files with a terminal \n that cannot be deleted. It is + invisible in the sense that one cannot move the cursor beyond it. + + This class is only tested (and valid) with strings of ascii chars. + For testing, we are not concerned with Tk Text's treatment of, + for instance, 0-width characters or character + accent. + """ + def __init__(self, master=None, cnf={}, **kw): + '''Initialize mock, non-gui, text-only Text widget. + + At present, all args are ignored. Almost all affect visual behavior. + There are just a few Text-only options that affect text behavior. + ''' + self.data = ['', '\n'] + + def index(self, index): + "Return string version of index decoded according to current text." + return "%s.%s" % self._decode(index, endflag=1) + + def _decode(self, index, endflag=0): + """Return a (line, char) tuple of int indexes into self.data. + + This implements .index without converting the result back to a string. + The result is contrained by the number of lines and linelengths of + self.data. For many indexes, the result is initially (1, 0). + + The input index may have any of several possible forms: + * line.char float: converted to 'line.char' string; + * 'line.char' string, where line and char are decimal integers; + * 'line.char lineend', where lineend='lineend' (and char is ignored); + * 'line.end', where end='end' (same as above); + * 'insert', the positions before terminal \n; + * 'end', whose meaning depends on the endflag passed to ._endex. + * 'sel.first' or 'sel.last', where sel is a tag -- not implemented. + """ + if isinstance(index, (float, bytes)): + index = str(index) + try: + index=index.lower() + except AttributeError: + raise TclError('bad text index "%s"' % index) + + lastline = len(self.data) - 1 # same as number of text lines + if index == 'insert': + return lastline, len(self.data[lastline]) - 1 + elif index == 'end': + return self._endex(endflag) + + line, char = index.split('.') + line = int(line) + + # Out of bounds line becomes first or last ('end') index + if line < 1: + return 1, 0 + elif line > lastline: + return self._endex(endflag) + + linelength = len(self.data[line]) -1 # position before/at \n + if char.endswith(' lineend') or char == 'end': + return line, linelength + # Tk requires that ignored chars before ' lineend' be valid int + + # Out of bounds char becomes first or last index of line + char = int(char) + if char < 0: + char = 0 + elif char > linelength: + char = linelength + return line, char + + def _endex(self, endflag): + '''Return position for 'end' or line overflow corresponding to endflag. + + -1: position before terminal \n; for .insert(), .delete + 0: position after terminal \n; for .get, .delete index 1 + 1: same viewed as beginning of non-existent next line (for .index) + ''' + n = len(self.data) + if endflag == 1: + return n, 0 + else: + n -= 1 + return n, len(self.data[n]) + endflag + + + def insert(self, index, chars): + "Insert chars before the character at index." + + if not chars: # ''.splitlines() is [], not [''] + return + chars = chars.splitlines(True) + if chars[-1][-1] == '\n': + chars.append('') + line, char = self._decode(index, -1) + before = self.data[line][:char] + after = self.data[line][char:] + self.data[line] = before + chars[0] + self.data[line+1:line+1] = chars[1:] + self.data[line+len(chars)-1] += after + + + def get(self, index1, index2=None): + "Return slice from index1 to index2 (default is 'index1+1')." + + startline, startchar = self._decode(index1) + if index2 is None: + endline, endchar = startline, startchar+1 + else: + endline, endchar = self._decode(index2) + + if startline == endline: + return self.data[startline][startchar:endchar] + else: + lines = [self.data[startline][startchar:]] + for i in range(startline+1, endline): + lines.append(self.data[i]) + lines.append(self.data[endline][:endchar]) + return ''.join(lines) + + + def delete(self, index1, index2=None): + '''Delete slice from index1 to index2 (default is 'index1+1'). + + Adjust default index2 ('index+1) for line ends. + Do not delete the terminal \n at the very end of self.data ([-1][-1]). + ''' + startline, startchar = self._decode(index1, -1) + if index2 is None: + if startchar < len(self.data[startline])-1: + # not deleting \n + endline, endchar = startline, startchar+1 + elif startline < len(self.data) - 1: + # deleting non-terminal \n, convert 'index1+1 to start of next line + endline, endchar = startline+1, 0 + else: + # do not delete terminal \n if index1 == 'insert' + return + else: + endline, endchar = self._decode(index2, -1) + # restricting end position to insert position excludes terminal \n + + if startline == endline and startchar < endchar: + self.data[startline] = self.data[startline][:startchar] + \ + self.data[startline][endchar:] + elif startline < endline: + self.data[startline] = self.data[startline][:startchar] + \ + self.data[endline][endchar:] + startline += 1 + for i in range(startline, endline+1): + del self.data[startline] + + def compare(self, index1, op, index2): + line1, char1 = self._decode(index1) + line2, char2 = self._decode(index2) + if op == '<': + return line1 < line2 or line1 == line2 and char1 < char2 + elif op == '<=': + return line1 < line2 or line1 == line2 and char1 <= char2 + elif op == '>': + return line1 > line2 or line1 == line2 and char1 > char2 + elif op == '>=': + return line1 > line2 or line1 == line2 and char1 >= char2 + elif op == '==': + return line1 == line2 and char1 == char2 + elif op == '!=': + return line1 != line2 or char1 != char2 + else: + raise TclError('''bad comparison operator "%s":''' + '''must be <, <=, ==, >=, >, or !=''' % op) + + # The following Text methods normally do something and return None. + # Whether doing nothing is sufficient for a test will depend on the test. + + def mark_set(self, name, index): + "Set mark *name* before the character at index." + pass + + def mark_unset(self, *markNames): + "Delete all marks in markNames." + + def tag_remove(self, tagName, index1, index2=None): + "Remove tag tagName from all characters between index1 and index2." + pass + + # The following Text methods affect the graphics screen and return None. + # Doing nothing should always be sufficient for tests. + + def scan_dragto(self, x, y): + "Adjust the view of the text according to scan_mark" + + def scan_mark(self, x, y): + "Remember the current X, Y coordinates." + + def see(self, index): + "Scroll screen to make the character at INDEX is visible." + pass + + # The following is a Misc method inherited by Text. + # It should properly go in a Misc mock, but is included here for now. + + def bind(sequence=None, func=None, add=None): + "Bind to this widget at event sequence a call to function func." + pass diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py new file mode 100644 index 00000000000..79884f21a6f --- /dev/null +++ b/Lib/idlelib/idle_test/test_calltips.py @@ -0,0 +1,20 @@ +import unittest +import idlelib.CallTips as ct +CTi = ct.CallTips() + +class Get_entityTest(unittest.TestCase): + # In 3.x, get_entity changed from 'instance method' to module function + # since 'self' not used. Use dummy instance until change 2.7 also. + def test_bad_entity(self): + self.assertIsNone(CTi.get_entity('1/0')) + def test_good_entity(self): + self.assertIs(CTi.get_entity('int'), int) + +class Py2Test(unittest.TestCase): + def test_paramtuple_float(self): + # 18539: (a,b) becomes '.0' in code object; change that but not float + def f((a,b), c=0.0): pass + self.assertEqual(ct.get_arg_text(f), '(, c=0.0)') + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_config_name.py b/Lib/idlelib/idle_test/test_config_name.py new file mode 100644 index 00000000000..4403f87fd3b --- /dev/null +++ b/Lib/idlelib/idle_test/test_config_name.py @@ -0,0 +1,75 @@ +"""Unit tests for idlelib.configSectionNameDialog""" +import unittest +from idlelib.idle_test.mock_tk import Var, Mbox +from idlelib import configSectionNameDialog as name_dialog_module + +name_dialog = name_dialog_module.GetCfgSectionNameDialog + +class Dummy_name_dialog(object): + # Mock for testing the following methods of name_dialog + name_ok = name_dialog.name_ok.im_func + Ok = name_dialog.Ok.im_func + Cancel = name_dialog.Cancel.im_func + # Attributes, constant or variable, needed for tests + used_names = ['used'] + name = Var() + result = None + destroyed = False + def destroy(self): + self.destroyed = True + +# name_ok calls Mbox.showerror if name is not ok +orig_mbox = name_dialog_module.tkMessageBox +showerror = Mbox.showerror + +class ConfigNameTest(unittest.TestCase): + dialog = Dummy_name_dialog() + + @classmethod + def setUpClass(cls): + name_dialog_module.tkMessageBox = Mbox + + @classmethod + def tearDownClass(cls): + name_dialog_module.tkMessageBox = orig_mbox + + def test_blank_name(self): + self.dialog.name.set(' ') + self.assertEqual(self.dialog.name_ok(), '') + self.assertEqual(showerror.title, 'Name Error') + self.assertIn('No', showerror.message) + + def test_used_name(self): + self.dialog.name.set('used') + self.assertEqual(self.dialog.name_ok(), '') + self.assertEqual(showerror.title, 'Name Error') + self.assertIn('use', showerror.message) + + def test_long_name(self): + self.dialog.name.set('good'*8) + self.assertEqual(self.dialog.name_ok(), '') + self.assertEqual(showerror.title, 'Name Error') + self.assertIn('too long', showerror.message) + + def test_good_name(self): + self.dialog.name.set(' good ') + showerror.title = 'No Error' # should not be called + self.assertEqual(self.dialog.name_ok(), 'good') + self.assertEqual(showerror.title, 'No Error') + + def test_ok(self): + self.dialog.destroyed = False + self.dialog.name.set('good') + self.dialog.Ok() + self.assertEqual(self.dialog.result, 'good') + self.assertTrue(self.dialog.destroyed) + + def test_cancel(self): + self.dialog.destroyed = False + self.dialog.Cancel() + self.assertEqual(self.dialog.result, '') + self.assertTrue(self.dialog.destroyed) + + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_delegator.py b/Lib/idlelib/idle_test/test_delegator.py new file mode 100644 index 00000000000..b8ae5eeefe3 --- /dev/null +++ b/Lib/idlelib/idle_test/test_delegator.py @@ -0,0 +1,37 @@ +import unittest +from idlelib.Delegator import Delegator + +class DelegatorTest(unittest.TestCase): + + def test_mydel(self): + # test a simple use scenario + + # initialize + mydel = Delegator(int) + self.assertIs(mydel.delegate, int) + self.assertEqual(mydel._Delegator__cache, set()) + + # add an attribute: + self.assertRaises(AttributeError, mydel.__getattr__, 'xyz') + bl = mydel.bit_length + self.assertIs(bl, int.bit_length) + self.assertIs(mydel.__dict__['bit_length'], int.bit_length) + self.assertEqual(mydel._Delegator__cache, {'bit_length'}) + + # add a second attribute + mydel.numerator + self.assertEqual(mydel._Delegator__cache, {'bit_length', 'numerator'}) + + # delete the second (which, however, leaves it in the name cache) + del mydel.numerator + self.assertNotIn('numerator', mydel.__dict__) + self.assertIn('numerator', mydel._Delegator__cache) + + # reset by calling .setdelegate, which calls .resetcache + mydel.setdelegate(float) + self.assertIs(mydel.delegate, float) + self.assertNotIn('bit_length', mydel.__dict__) + self.assertEqual(mydel._Delegator__cache, set()) + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=2) diff --git a/Lib/idlelib/idle_test/test_formatparagraph.py b/Lib/idlelib/idle_test/test_formatparagraph.py new file mode 100644 index 00000000000..7d7affc81ce --- /dev/null +++ b/Lib/idlelib/idle_test/test_formatparagraph.py @@ -0,0 +1,374 @@ +# Test the functions and main class method of FormatParagraph.py +import unittest +from idlelib import FormatParagraph as fp +from idlelib.EditorWindow import EditorWindow +from Tkinter import Tk, Text, TclError +from test.test_support import requires + + +class Is_Get_Test(unittest.TestCase): + """Test the is_ and get_ functions""" + test_comment = '# This is a comment' + test_nocomment = 'This is not a comment' + trailingws_comment = '# This is a comment ' + leadingws_comment = ' # This is a comment' + leadingws_nocomment = ' This is not a comment' + + def test_is_all_white(self): + self.assertTrue(fp.is_all_white('')) + self.assertTrue(fp.is_all_white('\t\n\r\f\v')) + self.assertFalse(fp.is_all_white(self.test_comment)) + + def test_get_indent(self): + Equal = self.assertEqual + Equal(fp.get_indent(self.test_comment), '') + Equal(fp.get_indent(self.trailingws_comment), '') + Equal(fp.get_indent(self.leadingws_comment), ' ') + Equal(fp.get_indent(self.leadingws_nocomment), ' ') + + def test_get_comment_header(self): + Equal = self.assertEqual + # Test comment strings + Equal(fp.get_comment_header(self.test_comment), '#') + Equal(fp.get_comment_header(self.trailingws_comment), '#') + Equal(fp.get_comment_header(self.leadingws_comment), ' #') + # Test non-comment strings + Equal(fp.get_comment_header(self.leadingws_nocomment), ' ') + Equal(fp.get_comment_header(self.test_nocomment), '') + + +class FindTest(unittest.TestCase): + """Test the find_paragraph function in FormatParagraph. + + Using the runcase() function, find_paragraph() is called with 'mark' set at + multiple indexes before and inside the test paragraph. + + It appears that code with the same indentation as a quoted string is grouped + as part of the same paragraph, which is probably incorrect behavior. + """ + + @classmethod + def setUpClass(cls): + from idlelib.idle_test.mock_tk import Text + cls.text = Text() + + def runcase(self, inserttext, stopline, expected): + # Check that find_paragraph returns the expected paragraph when + # the mark index is set to beginning, middle, end of each line + # up to but not including the stop line + text = self.text + text.insert('1.0', inserttext) + for line in range(1, stopline): + linelength = int(text.index("%d.end" % line).split('.')[1]) + for col in (0, linelength//2, linelength): + tempindex = "%d.%d" % (line, col) + self.assertEqual(fp.find_paragraph(text, tempindex), expected) + text.delete('1.0', 'end') + + def test_find_comment(self): + comment = ( + "# Comment block with no blank lines before\n" + "# Comment line\n" + "\n") + self.runcase(comment, 3, ('1.0', '3.0', '#', comment[0:58])) + + comment = ( + "\n" + "# Comment block with whitespace line before and after\n" + "# Comment line\n" + "\n") + self.runcase(comment, 4, ('2.0', '4.0', '#', comment[1:70])) + + comment = ( + "\n" + " # Indented comment block with whitespace before and after\n" + " # Comment line\n" + "\n") + self.runcase(comment, 4, ('2.0', '4.0', ' #', comment[1:82])) + + comment = ( + "\n" + "# Single line comment\n" + "\n") + self.runcase(comment, 3, ('2.0', '3.0', '#', comment[1:23])) + + comment = ( + "\n" + " # Single line comment with leading whitespace\n" + "\n") + self.runcase(comment, 3, ('2.0', '3.0', ' #', comment[1:51])) + + comment = ( + "\n" + "# Comment immediately followed by code\n" + "x = 42\n" + "\n") + self.runcase(comment, 3, ('2.0', '3.0', '#', comment[1:40])) + + comment = ( + "\n" + " # Indented comment immediately followed by code\n" + "x = 42\n" + "\n") + self.runcase(comment, 3, ('2.0', '3.0', ' #', comment[1:53])) + + comment = ( + "\n" + "# Comment immediately followed by indented code\n" + " x = 42\n" + "\n") + self.runcase(comment, 3, ('2.0', '3.0', '#', comment[1:49])) + + def test_find_paragraph(self): + teststring = ( + '"""String with no blank lines before\n' + 'String line\n' + '"""\n' + '\n') + self.runcase(teststring, 4, ('1.0', '4.0', '', teststring[0:53])) + + teststring = ( + "\n" + '"""String with whitespace line before and after\n' + 'String line.\n' + '"""\n' + '\n') + self.runcase(teststring, 5, ('2.0', '5.0', '', teststring[1:66])) + + teststring = ( + '\n' + ' """Indented string with whitespace before and after\n' + ' Comment string.\n' + ' """\n' + '\n') + self.runcase(teststring, 5, ('2.0', '5.0', ' ', teststring[1:85])) + + teststring = ( + '\n' + '"""Single line string."""\n' + '\n') + self.runcase(teststring, 3, ('2.0', '3.0', '', teststring[1:27])) + + teststring = ( + '\n' + ' """Single line string with leading whitespace."""\n' + '\n') + self.runcase(teststring, 3, ('2.0', '3.0', ' ', teststring[1:55])) + + +class ReformatFunctionTest(unittest.TestCase): + """Test the reformat_paragraph function without the editor window.""" + + def test_reformat_paragrah(self): + Equal = self.assertEqual + reform = fp.reformat_paragraph + hw = "O hello world" + Equal(reform(' ', 1), ' ') + Equal(reform("Hello world", 20), "Hello world") + + # Test without leading newline + Equal(reform(hw, 1), "O\nhello\nworld") + Equal(reform(hw, 6), "O\nhello\nworld") + Equal(reform(hw, 7), "O hello\nworld") + Equal(reform(hw, 12), "O hello\nworld") + Equal(reform(hw, 13), "O hello world") + + # Test with leading newline + hw = "\nO hello world" + Equal(reform(hw, 1), "\nO\nhello\nworld") + Equal(reform(hw, 6), "\nO\nhello\nworld") + Equal(reform(hw, 7), "\nO hello\nworld") + Equal(reform(hw, 12), "\nO hello\nworld") + Equal(reform(hw, 13), "\nO hello world") + + +class ReformatCommentTest(unittest.TestCase): + """Test the reformat_comment function without the editor window.""" + + def test_reformat_comment(self): + Equal = self.assertEqual + + # reformat_comment formats to a minimum of 20 characters + test_string = ( + " \"\"\"this is a test of a reformat for a triple quoted string" + " will it reformat to less than 70 characters for me?\"\"\"") + result = fp.reformat_comment(test_string, 70, " ") + expected = ( + " \"\"\"this is a test of a reformat for a triple quoted string will it\n" + " reformat to less than 70 characters for me?\"\"\"") + Equal(result, expected) + + test_comment = ( + "# this is a test of a reformat for a triple quoted string will " + "it reformat to less than 70 characters for me?") + result = fp.reformat_comment(test_comment, 70, "#") + expected = ( + "# this is a test of a reformat for a triple quoted string will it\n" + "# reformat to less than 70 characters for me?") + Equal(result, expected) + + +class FormatClassTest(unittest.TestCase): + def test_init_close(self): + instance = fp.FormatParagraph('editor') + self.assertEqual(instance.editwin, 'editor') + instance.close() + self.assertEqual(instance.editwin, None) + + +# For testing format_paragraph_event, Initialize FormatParagraph with +# a mock Editor with .text and .get_selection_indices. The text must +# be a Text wrapper that adds two methods + +# A real EditorWindow creates unneeded, time-consuming baggage and +# sometimes emits shutdown warnings like this: +# "warning: callback failed in WindowList +# : invalid command name ".55131368.windows". +# Calling EditorWindow._close in tearDownClass prevents this but causes +# other problems (windows left open). + +class TextWrapper: + def __init__(self, master): + self.text = Text(master=master) + def __getattr__(self, name): + return getattr(self.text, name) + def undo_block_start(self): pass + def undo_block_stop(self): pass + +class Editor: + def __init__(self, root): + self.text = TextWrapper(root) + get_selection_indices = EditorWindow. get_selection_indices.im_func + +class FormatEventTest(unittest.TestCase): + """Test the formatting of text inside a Text widget. + + This is done with FormatParagraph.format.paragraph_event, + which calls functions in the module as appropriate. + """ + test_string = ( + " '''this is a test of a reformat for a triple " + "quoted string will it reformat to less than 70 " + "characters for me?'''\n") + multiline_test_string = ( + " '''The first line is under the max width.\n" + " The second line's length is way over the max width. It goes " + "on and on until it is over 100 characters long.\n" + " Same thing with the third line. It is also way over the max " + "width, but FormatParagraph will fix it.\n" + " '''\n") + multiline_test_comment = ( + "# The first line is under the max width.\n" + "# The second line's length is way over the max width. It goes on " + "and on until it is over 100 characters long.\n" + "# Same thing with the third line. It is also way over the max " + "width, but FormatParagraph will fix it.\n" + "# The fourth line is short like the first line.") + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = Tk() + editor = Editor(root=cls.root) + cls.text = editor.text.text # Test code does not need the wrapper. + cls.formatter = fp.FormatParagraph(editor).format_paragraph_event + # Sets the insert mark just after the re-wrapped and inserted text. + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + + def test_short_line(self): + self.text.insert('1.0', "Short line\n") + self.formatter("Dummy") + self.assertEqual(self.text.get('1.0', 'insert'), "Short line\n" ) + self.text.delete('1.0', 'end') + + def test_long_line(self): + text = self.text + + # Set cursor ('insert' mark) to '1.0', within text. + text.insert('1.0', self.test_string) + text.mark_set('insert', '1.0') + self.formatter('ParameterDoesNothing') + result = text.get('1.0', 'insert') + # find function includes \n + expected = ( +" '''this is a test of a reformat for a triple quoted string will it\n" +" reformat to less than 70 characters for me?'''\n") # yes + self.assertEqual(result, expected) + text.delete('1.0', 'end') + + # Select from 1.11 to line end. + text.insert('1.0', self.test_string) + text.tag_add('sel', '1.11', '1.end') + self.formatter('ParameterDoesNothing') + result = text.get('1.0', 'insert') + # selection excludes \n + expected = ( +" '''this is a test of a reformat for a triple quoted string will it reformat\n" +" to less than 70 characters for me?'''") # no + self.assertEqual(result, expected) + text.delete('1.0', 'end') + + def test_multiple_lines(self): + text = self.text + # Select 2 long lines. + text.insert('1.0', self.multiline_test_string) + text.tag_add('sel', '2.0', '4.0') + self.formatter('ParameterDoesNothing') + result = text.get('2.0', 'insert') + expected = ( +" The second line's length is way over the max width. It goes on and\n" +" on until it is over 100 characters long. Same thing with the third\n" +" line. It is also way over the max width, but FormatParagraph will\n" +" fix it.\n") + self.assertEqual(result, expected) + text.delete('1.0', 'end') + + def test_comment_block(self): + text = self.text + + # Set cursor ('insert') to '1.0', within block. + text.insert('1.0', self.multiline_test_comment) + self.formatter('ParameterDoesNothing') + result = text.get('1.0', 'insert') + expected = ( +"# The first line is under the max width. The second line's length is\n" +"# way over the max width. It goes on and on until it is over 100\n" +"# characters long. Same thing with the third line. It is also way over\n" +"# the max width, but FormatParagraph will fix it. The fourth line is\n" +"# short like the first line.\n") + self.assertEqual(result, expected) + text.delete('1.0', 'end') + + # Select line 2, verify line 1 unaffected. + text.insert('1.0', self.multiline_test_comment) + text.tag_add('sel', '2.0', '3.0') + self.formatter('ParameterDoesNothing') + result = text.get('1.0', 'insert') + expected = ( +"# The first line is under the max width.\n" +"# The second line's length is way over the max width. It goes on and\n" +"# on until it is over 100 characters long.\n") + self.assertEqual(result, expected) + text.delete('1.0', 'end') + +# The following block worked with EditorWindow but fails with the mock. +# Lines 2 and 3 get pasted together even though the previous block left +# the previous line alone. More investigation is needed. +## # Select lines 3 and 4 +## text.insert('1.0', self.multiline_test_comment) +## text.tag_add('sel', '3.0', '5.0') +## self.formatter('ParameterDoesNothing') +## result = text.get('3.0', 'insert') +## expected = ( +##"# Same thing with the third line. It is also way over the max width,\n" +##"# but FormatParagraph will fix it. The fourth line is short like the\n" +##"# first line.\n") +## self.assertEqual(result, expected) +## text.delete('1.0', 'end') + + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=2) diff --git a/Lib/idlelib/idle_test/test_grep.py b/Lib/idlelib/idle_test/test_grep.py new file mode 100644 index 00000000000..e9f4f22ae65 --- /dev/null +++ b/Lib/idlelib/idle_test/test_grep.py @@ -0,0 +1,82 @@ +""" !Changing this line will break Test_findfile.test_found! +Non-gui unit tests for idlelib.GrepDialog methods. +dummy_command calls grep_it calls findfiles. +An exception raised in one method will fail callers. +Otherwise, tests are mostly independent. +*** Currently only test grep_it. +""" +import unittest +from test.test_support import captured_stdout, findfile +from idlelib.idle_test.mock_tk import Var +from idlelib.GrepDialog import GrepDialog +import re + +__file__ = findfile('idlelib/idle_test') + '/test_grep.py' + +class Dummy_searchengine: + '''GrepDialog.__init__ calls parent SearchDiabolBase which attaches the + passed in SearchEngine instance as attribute 'engine'. Only a few of the + many possible self.engine.x attributes are needed here. + ''' + def getpat(self): + return self._pat + +searchengine = Dummy_searchengine() + +class Dummy_grep: + # Methods tested + #default_command = GrepDialog.default_command + grep_it = GrepDialog.grep_it.im_func + findfiles = GrepDialog.findfiles.im_func + # Other stuff needed + recvar = Var(False) + engine = searchengine + def close(self): # gui method + pass + +grep = Dummy_grep() + +class FindfilesTest(unittest.TestCase): + # findfiles is really a function, not a method, could be iterator + # test that filename return filename + # test that idlelib has many .py files + # test that recursive flag adds idle_test .py files + pass + +class Grep_itTest(unittest.TestCase): + # Test captured reports with 0 and some hits. + # Should test file names, but Windows reports have mixed / and \ separators + # from incomplete replacement, so 'later'. + + def report(self, pat): + grep.engine._pat = pat + with captured_stdout() as s: + grep.grep_it(re.compile(pat), __file__) + lines = s.getvalue().split('\n') + lines.pop() # remove bogus '' after last \n + return lines + + def test_unfound(self): + pat = 'xyz*'*7 + lines = self.report(pat) + self.assertEqual(len(lines), 2) + self.assertIn(pat, lines[0]) + self.assertEqual(lines[1], 'No hits.') + + def test_found(self): + + pat = '""" !Changing this line will break Test_findfile.test_found!' + lines = self.report(pat) + self.assertEqual(len(lines), 5) + self.assertIn(pat, lines[0]) + self.assertIn('py: 1:', lines[1]) # line number 1 + self.assertIn('2', lines[3]) # hits found 2 + self.assertTrue(lines[4].startswith('(Hint:')) + +class Default_commandTest(unittest.TestCase): + # To write this, mode OutputWindow import to top of GrepDialog + # so it can be replaced by captured_stdout in class setup/teardown. + pass + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_idlehistory.py b/Lib/idlelib/idle_test/test_idlehistory.py new file mode 100644 index 00000000000..c15a9266d77 --- /dev/null +++ b/Lib/idlelib/idle_test/test_idlehistory.py @@ -0,0 +1,166 @@ +import unittest +from test.test_support import requires + +import Tkinter as tk +from Tkinter import Text as tkText +from idlelib.idle_test.mock_tk import Text as mkText +from idlelib.IdleHistory import History +from idlelib.configHandler import idleConf + +line1 = 'a = 7' +line2 = 'b = a' + +class StoreTest(unittest.TestCase): + '''Tests History.__init__ and History.store with mock Text''' + + @classmethod + def setUpClass(cls): + cls.text = mkText() + cls.history = History(cls.text) + + def tearDown(self): + self.text.delete('1.0', 'end') + self.history.history = [] + + def test_init(self): + self.assertIs(self.history.text, self.text) + self.assertEqual(self.history.history, []) + self.assertIsNone(self.history.prefix) + self.assertIsNone(self.history.pointer) + self.assertEqual(self.history.cyclic, + idleConf.GetOption("main", "History", "cyclic", 1, "bool")) + + def test_store_short(self): + self.history.store('a') + self.assertEqual(self.history.history, []) + self.history.store(' a ') + self.assertEqual(self.history.history, []) + + def test_store_dup(self): + self.history.store(line1) + self.assertEqual(self.history.history, [line1]) + self.history.store(line2) + self.assertEqual(self.history.history, [line1, line2]) + self.history.store(line1) + self.assertEqual(self.history.history, [line2, line1]) + + def test_store_reset(self): + self.history.prefix = line1 + self.history.pointer = 0 + self.history.store(line2) + self.assertIsNone(self.history.prefix) + self.assertIsNone(self.history.pointer) + + +class TextWrapper: + def __init__(self, master): + self.text = tkText(master=master) + self._bell = False + def __getattr__(self, name): + return getattr(self.text, name) + def bell(self): + self._bell = True + +class FetchTest(unittest.TestCase): + '''Test History.fetch with wrapped tk.Text. + ''' + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = tk.Tk() + + def setUp(self): + self.text = text = TextWrapper(self.root) + text.insert('1.0', ">>> ") + text.mark_set('iomark', '1.4') + text.mark_gravity('iomark', 'left') + self.history = History(text) + self.history.history = [line1, line2] + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + + def fetch_test(self, reverse, line, prefix, index, bell=False): + # Perform one fetch as invoked by Alt-N or Alt-P + # Test the result. The line test is the most important. + # The last two are diagnostic of fetch internals. + History = self.history + History.fetch(reverse) + + Equal = self.assertEqual + Equal(self.text.get('iomark', 'end-1c'), line) + Equal(self.text._bell, bell) + if bell: + self.text._bell = False + Equal(History.prefix, prefix) + Equal(History.pointer, index) + Equal(self.text.compare("insert", '==', "end-1c"), 1) + + def test_fetch_prev_cyclic(self): + prefix = '' + test = self.fetch_test + test(True, line2, prefix, 1) + test(True, line1, prefix, 0) + test(True, prefix, None, None, bell=True) + + def test_fetch_next_cyclic(self): + prefix = '' + test = self.fetch_test + test(False, line1, prefix, 0) + test(False, line2, prefix, 1) + test(False, prefix, None, None, bell=True) + + # Prefix 'a' tests skip line2, which starts with 'b' + def test_fetch_prev_prefix(self): + prefix = 'a' + self.text.insert('iomark', prefix) + self.fetch_test(True, line1, prefix, 0) + self.fetch_test(True, prefix, None, None, bell=True) + + def test_fetch_next_prefix(self): + prefix = 'a' + self.text.insert('iomark', prefix) + self.fetch_test(False, line1, prefix, 0) + self.fetch_test(False, prefix, None, None, bell=True) + + def test_fetch_prev_noncyclic(self): + prefix = '' + self.history.cyclic = False + test = self.fetch_test + test(True, line2, prefix, 1) + test(True, line1, prefix, 0) + test(True, line1, prefix, 0, bell=True) + + def test_fetch_next_noncyclic(self): + prefix = '' + self.history.cyclic = False + test = self.fetch_test + test(False, prefix, None, None, bell=True) + test(True, line2, prefix, 1) + test(False, prefix, None, None, bell=True) + test(False, prefix, None, None, bell=True) + + def test_fetch_cursor_move(self): + # Move cursor after fetch + self.history.fetch(reverse=True) # initialization + self.text.mark_set('insert', 'iomark') + self.fetch_test(True, line2, None, None, bell=True) + + def test_fetch_edit(self): + # Edit after fetch + self.history.fetch(reverse=True) # initialization + self.text.delete('iomark', 'insert', ) + self.text.insert('iomark', 'a =') + self.fetch_test(True, line1, 'a =', 0) # prefix is reset + + def test_history_prev_next(self): + # Minimally test functions bound to events + self.history.history_prev('dummy event') + self.assertEqual(self.history.pointer, 1) + self.history.history_next('dummy event') + self.assertEqual(self.history.pointer, None) + + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=2) diff --git a/Lib/idlelib/idle_test/test_pathbrowser.py b/Lib/idlelib/idle_test/test_pathbrowser.py new file mode 100644 index 00000000000..7ad7c97a79e --- /dev/null +++ b/Lib/idlelib/idle_test/test_pathbrowser.py @@ -0,0 +1,12 @@ +import unittest +import idlelib.PathBrowser as PathBrowser + +class PathBrowserTest(unittest.TestCase): + + def test_DirBrowserTreeItem(self): + # Issue16226 - make sure that getting a sublist works + d = PathBrowser.DirBrowserTreeItem('') + d.GetSubList() + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_rstrip.py b/Lib/idlelib/idle_test/test_rstrip.py new file mode 100644 index 00000000000..1c90b93d216 --- /dev/null +++ b/Lib/idlelib/idle_test/test_rstrip.py @@ -0,0 +1,49 @@ +import unittest +import idlelib.RstripExtension as rs +from idlelib.idle_test.mock_idle import Editor + +class rstripTest(unittest.TestCase): + + def test_rstrip_line(self): + editor = Editor() + text = editor.text + do_rstrip = rs.RstripExtension(editor).do_rstrip + + do_rstrip() + self.assertEqual(text.get('1.0', 'insert'), '') + text.insert('1.0', ' ') + do_rstrip() + self.assertEqual(text.get('1.0', 'insert'), '') + text.insert('1.0', ' \n') + do_rstrip() + self.assertEqual(text.get('1.0', 'insert'), '\n') + + def test_rstrip_multiple(self): + editor = Editor() + # Uncomment following to verify that test passes with real widgets. +## from idlelib.EditorWindow import EditorWindow as Editor +## from tkinter import Tk +## editor = Editor(root=Tk()) + text = editor.text + do_rstrip = rs.RstripExtension(editor).do_rstrip + + original = ( + "Line with an ending tab \n" + "Line ending in 5 spaces \n" + "Linewithnospaces\n" + " indented line\n" + " indented line with trailing space \n" + " ") + stripped = ( + "Line with an ending tab\n" + "Line ending in 5 spaces\n" + "Linewithnospaces\n" + " indented line\n" + " indented line with trailing space\n") + + text.insert('1.0', original) + do_rstrip() + self.assertEqual(text.get('1.0', 'insert'), stripped) + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_searchengine.py b/Lib/idlelib/idle_test/test_searchengine.py new file mode 100644 index 00000000000..e5e720de956 --- /dev/null +++ b/Lib/idlelib/idle_test/test_searchengine.py @@ -0,0 +1,326 @@ +'''Test functions and SearchEngine class in SearchEngine.py.''' + +# With mock replacements, the module does not use any gui widgets. +# The use of tk.Text is avoided (for now, until mock Text is improved) +# by patching instances with an index function returning what is needed. +# This works because mock Text.get does not use .index. + +import re +import unittest +from test.test_support import requires +from Tkinter import BooleanVar, StringVar, TclError # ,Tk, Text +import tkMessageBox +from idlelib import SearchEngine as se +from idlelib.idle_test.mock_tk import Var, Mbox +from idlelib.idle_test.mock_tk import Text as mockText + +def setUpModule(): + # Replace s-e module tkinter imports other than non-gui TclError. + se.BooleanVar = Var + se.StringVar = Var + se.tkMessageBox = Mbox + +def tearDownModule(): + # Restore 'just in case', though other tests should also replace. + se.BooleanVar = BooleanVar + se.StringVar = StringVar + se.tkMessageBox = tkMessageBox + + +class Mock: + def __init__(self, *args, **kwargs): pass + +class GetTest(unittest.TestCase): + # SearchEngine.get returns singleton created & saved on first call. + def test_get(self): + saved_Engine = se.SearchEngine + se.SearchEngine = Mock # monkey-patch class + try: + root = Mock() + engine = se.get(root) + self.assertIsInstance(engine, se.SearchEngine) + self.assertIs(root._searchengine, engine) + self.assertIs(se.get(root), engine) + finally: + se.SearchEngine = saved_Engine # restore class to module + +class GetLineColTest(unittest.TestCase): + # Test simple text-independent helper function + def test_get_line_col(self): + self.assertEqual(se.get_line_col('1.0'), (1, 0)) + self.assertEqual(se.get_line_col('1.11'), (1, 11)) + + self.assertRaises(ValueError, se.get_line_col, ('1.0 lineend')) + self.assertRaises(ValueError, se.get_line_col, ('end')) + +class GetSelectionTest(unittest.TestCase): + # Test text-dependent helper function. +## # Need gui for text.index('sel.first/sel.last/insert'). +## @classmethod +## def setUpClass(cls): +## requires('gui') +## cls.root = Tk() +## +## @classmethod +## def tearDownClass(cls): +## cls.root.destroy() + + def test_get_selection(self): + # text = Text(master=self.root) + text = mockText() + text.insert('1.0', 'Hello World!') + + # fix text.index result when called in get_selection + def sel(s): + # select entire text, cursor irrelevant + if s == 'sel.first': return '1.0' + if s == 'sel.last': return '1.12' + raise TclError + text.index = sel # replaces .tag_add('sel', '1.0, '1.12') + self.assertEqual(se.get_selection(text), ('1.0', '1.12')) + + def mark(s): + # no selection, cursor after 'Hello' + if s == 'insert': return '1.5' + raise TclError + text.index = mark # replaces .mark_set('insert', '1.5') + self.assertEqual(se.get_selection(text), ('1.5', '1.5')) + + +class ReverseSearchTest(unittest.TestCase): + # Test helper function that searches backwards within a line. + def test_search_reverse(self): + Equal = self.assertEqual + line = "Here is an 'is' test text." + prog = re.compile('is') + Equal(se.search_reverse(prog, line, len(line)).span(), (12, 14)) + Equal(se.search_reverse(prog, line, 14).span(), (12, 14)) + Equal(se.search_reverse(prog, line, 13).span(), (5, 7)) + Equal(se.search_reverse(prog, line, 7).span(), (5, 7)) + Equal(se.search_reverse(prog, line, 6), None) + + +class SearchEngineTest(unittest.TestCase): + # Test class methods that do not use Text widget. + + def setUp(self): + self.engine = se.SearchEngine(root=None) + # Engine.root is only used to create error message boxes. + # The mock replacement ignores the root argument. + + def test_is_get(self): + engine = self.engine + Equal = self.assertEqual + + Equal(engine.getpat(), '') + engine.setpat('hello') + Equal(engine.getpat(), 'hello') + + Equal(engine.isre(), False) + engine.revar.set(1) + Equal(engine.isre(), True) + + Equal(engine.iscase(), False) + engine.casevar.set(1) + Equal(engine.iscase(), True) + + Equal(engine.isword(), False) + engine.wordvar.set(1) + Equal(engine.isword(), True) + + Equal(engine.iswrap(), True) + engine.wrapvar.set(0) + Equal(engine.iswrap(), False) + + Equal(engine.isback(), False) + engine.backvar.set(1) + Equal(engine.isback(), True) + + def test_setcookedpat(self): + engine = self.engine + engine.setcookedpat('\s') + self.assertEqual(engine.getpat(), '\s') + engine.revar.set(1) + engine.setcookedpat('\s') + self.assertEqual(engine.getpat(), r'\\s') + + def test_getcookedpat(self): + engine = self.engine + Equal = self.assertEqual + + Equal(engine.getcookedpat(), '') + engine.setpat('hello') + Equal(engine.getcookedpat(), 'hello') + engine.wordvar.set(True) + Equal(engine.getcookedpat(), r'\bhello\b') + engine.wordvar.set(False) + + engine.setpat('\s') + Equal(engine.getcookedpat(), r'\\s') + engine.revar.set(True) + Equal(engine.getcookedpat(), '\s') + + def test_getprog(self): + engine = self.engine + Equal = self.assertEqual + + engine.setpat('Hello') + temppat = engine.getprog() + Equal(temppat.pattern, re.compile('Hello', re.IGNORECASE).pattern) + engine.casevar.set(1) + temppat = engine.getprog() + Equal(temppat.pattern, re.compile('Hello').pattern, 0) + + engine.setpat('') + Equal(engine.getprog(), None) + engine.setpat('+') + engine.revar.set(1) + Equal(engine.getprog(), None) + self.assertEqual(Mbox.showerror.message, + 'Error: nothing to repeat\nPattern: +') + + def test_report_error(self): + showerror = Mbox.showerror + Equal = self.assertEqual + pat = '[a-z' + msg = 'unexpected end of regular expression' + + Equal(self.engine.report_error(pat, msg), None) + Equal(showerror.title, 'Regular expression error') + expected_message = ("Error: " + msg + "\nPattern: [a-z") + Equal(showerror.message, expected_message) + + Equal(self.engine.report_error(pat, msg, 5), None) + Equal(showerror.title, 'Regular expression error') + expected_message += "\nOffset: 5" + Equal(showerror.message, expected_message) + + +class SearchTest(unittest.TestCase): + # Test that search_text makes right call to right method. + + @classmethod + def setUpClass(cls): +## requires('gui') +## cls.root = Tk() +## cls.text = Text(master=cls.root) + cls.text = mockText() + test_text = ( + 'First line\n' + 'Line with target\n' + 'Last line\n') + cls.text.insert('1.0', test_text) + cls.pat = re.compile('target') + + cls.engine = se.SearchEngine(None) + cls.engine.search_forward = lambda *args: ('f', args) + cls.engine.search_backward = lambda *args: ('b', args) + +## @classmethod +## def tearDownClass(cls): +## cls.root.destroy() + + def test_search(self): + Equal = self.assertEqual + engine = self.engine + search = engine.search_text + text = self.text + pat = self.pat + + engine.patvar.set(None) + #engine.revar.set(pat) + Equal(search(text), None) + + def mark(s): + # no selection, cursor after 'Hello' + if s == 'insert': return '1.5' + raise TclError + text.index = mark + Equal(search(text, pat), ('f', (text, pat, 1, 5, True, False))) + engine.wrapvar.set(False) + Equal(search(text, pat), ('f', (text, pat, 1, 5, False, False))) + engine.wrapvar.set(True) + engine.backvar.set(True) + Equal(search(text, pat), ('b', (text, pat, 1, 5, True, False))) + engine.backvar.set(False) + + def sel(s): + if s == 'sel.first': return '2.10' + if s == 'sel.last': return '2.16' + raise TclError + text.index = sel + Equal(search(text, pat), ('f', (text, pat, 2, 16, True, False))) + Equal(search(text, pat, True), ('f', (text, pat, 2, 10, True, True))) + engine.backvar.set(True) + Equal(search(text, pat), ('b', (text, pat, 2, 10, True, False))) + Equal(search(text, pat, True), ('b', (text, pat, 2, 16, True, True))) + + +class ForwardBackwardTest(unittest.TestCase): + # Test that search_forward method finds the target. +## @classmethod +## def tearDownClass(cls): +## cls.root.destroy() + + @classmethod + def setUpClass(cls): + cls.engine = se.SearchEngine(None) +## requires('gui') +## cls.root = Tk() +## cls.text = Text(master=cls.root) + cls.text = mockText() + # search_backward calls index('end-1c') + cls.text.index = lambda index: '4.0' + test_text = ( + 'First line\n' + 'Line with target\n' + 'Last line\n') + cls.text.insert('1.0', test_text) + cls.pat = re.compile('target') + cls.res = (2, (10, 16)) # line, slice indexes of 'target' + cls.failpat = re.compile('xyz') # not in text + cls.emptypat = re.compile('\w*') # empty match possible + + def make_search(self, func): + def search(pat, line, col, wrap, ok=0): + res = func(self.text, pat, line, col, wrap, ok) + # res is (line, matchobject) or None + return (res[0], res[1].span()) if res else res + return search + + def test_search_forward(self): + # search for non-empty match + Equal = self.assertEqual + forward = self.make_search(self.engine.search_forward) + pat = self.pat + Equal(forward(pat, 1, 0, True), self.res) + Equal(forward(pat, 3, 0, True), self.res) # wrap + Equal(forward(pat, 3, 0, False), None) # no wrap + Equal(forward(pat, 2, 10, False), self.res) + + Equal(forward(self.failpat, 1, 0, True), None) + Equal(forward(self.emptypat, 2, 9, True, ok=True), (2, (9, 9))) + #Equal(forward(self.emptypat, 2, 9, True), self.res) + # While the initial empty match is correctly ignored, skipping + # the rest of the line and returning (3, (0,4)) seems buggy - tjr. + Equal(forward(self.emptypat, 2, 10, True), self.res) + + def test_search_backward(self): + # search for non-empty match + Equal = self.assertEqual + backward = self.make_search(self.engine.search_backward) + pat = self.pat + Equal(backward(pat, 3, 5, True), self.res) + Equal(backward(pat, 2, 0, True), self.res) # wrap + Equal(backward(pat, 2, 0, False), None) # no wrap + Equal(backward(pat, 2, 16, False), self.res) + + Equal(backward(self.failpat, 3, 9, True), None) + Equal(backward(self.emptypat, 2, 10, True, ok=True), (2, (9,9))) + # Accepted because 9 < 10, not because ok=True. + # It is not clear that ok=True is useful going back - tjr + Equal(backward(self.emptypat, 2, 9, True), (2, (5, 9))) + + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=2) diff --git a/Lib/idlelib/idle_test/test_text.py b/Lib/idlelib/idle_test/test_text.py new file mode 100644 index 00000000000..3a0705b943c --- /dev/null +++ b/Lib/idlelib/idle_test/test_text.py @@ -0,0 +1,227 @@ +# Test mock_tk.Text class against tkinter.Text class by running same tests with both. +import unittest +from test.test_support import requires + +from _tkinter import TclError +import Tkinter as tk + +class TextTest(object): + + hw = 'hello\nworld' # usual initial insert after initialization + hwn = hw+'\n' # \n present at initialization, before insert + + Text = None + def setUp(self): + self.text = self.Text() + + def test_init(self): + self.assertEqual(self.text.get('1.0'), '\n') + self.assertEqual(self.text.get('end'), '') + + def test_index_empty(self): + index = self.text.index + + for dex in (-1.0, 0.3, '1.-1', '1.0', '1.0 lineend', '1.end', '1.33', + 'insert'): + self.assertEqual(index(dex), '1.0') + + for dex in 'end', 2.0, '2.1', '33.44': + self.assertEqual(index(dex), '2.0') + + def test_index_data(self): + index = self.text.index + self.text.insert('1.0', self.hw) + + for dex in -1.0, 0.3, '1.-1', '1.0': + self.assertEqual(index(dex), '1.0') + + for dex in '1.0 lineend', '1.end', '1.33': + self.assertEqual(index(dex), '1.5') + + for dex in 'end', '33.44': + self.assertEqual(index(dex), '3.0') + + def test_get(self): + get = self.text.get + Equal = self.assertEqual + self.text.insert('1.0', self.hw) + + Equal(get('end'), '') + Equal(get('end', 'end'), '') + Equal(get('1.0'), 'h') + Equal(get('1.0', '1.1'), 'h') + Equal(get('1.0', '1.3'), 'hel') + Equal(get('1.1', '1.3'), 'el') + Equal(get('1.0', '1.0 lineend'), 'hello') + Equal(get('1.0', '1.10'), 'hello') + Equal(get('1.0 lineend'), '\n') + Equal(get('1.1', '2.3'), 'ello\nwor') + Equal(get('1.0', '2.5'), self.hw) + Equal(get('1.0', 'end'), self.hwn) + Equal(get('0.0', '5.0'), self.hwn) + + def test_insert(self): + insert = self.text.insert + get = self.text.get + Equal = self.assertEqual + + insert('1.0', self.hw) + Equal(get('1.0', 'end'), self.hwn) + + insert('1.0', '') # nothing + Equal(get('1.0', 'end'), self.hwn) + + insert('1.0', '*') + Equal(get('1.0', 'end'), '*hello\nworld\n') + + insert('1.0 lineend', '*') + Equal(get('1.0', 'end'), '*hello*\nworld\n') + + insert('2.3', '*') + Equal(get('1.0', 'end'), '*hello*\nwor*ld\n') + + insert('end', 'x') + Equal(get('1.0', 'end'), '*hello*\nwor*ldx\n') + + insert('1.4', 'x\n') + Equal(get('1.0', 'end'), '*helx\nlo*\nwor*ldx\n') + + def test_no_delete(self): + # if index1 == 'insert' or 'end' or >= end, there is no deletion + delete = self.text.delete + get = self.text.get + Equal = self.assertEqual + self.text.insert('1.0', self.hw) + + delete('insert') + Equal(get('1.0', 'end'), self.hwn) + + delete('end') + Equal(get('1.0', 'end'), self.hwn) + + delete('insert', 'end') + Equal(get('1.0', 'end'), self.hwn) + + delete('insert', '5.5') + Equal(get('1.0', 'end'), self.hwn) + + delete('1.4', '1.0') + Equal(get('1.0', 'end'), self.hwn) + + delete('1.4', '1.4') + Equal(get('1.0', 'end'), self.hwn) + + def test_delete_char(self): + delete = self.text.delete + get = self.text.get + Equal = self.assertEqual + self.text.insert('1.0', self.hw) + + delete('1.0') + Equal(get('1.0', '1.end'), 'ello') + + delete('1.0', '1.1') + Equal(get('1.0', '1.end'), 'llo') + + # delete \n and combine 2 lines into 1 + delete('1.end') + Equal(get('1.0', '1.end'), 'lloworld') + + self.text.insert('1.3', '\n') + delete('1.10') + Equal(get('1.0', '1.end'), 'lloworld') + + self.text.insert('1.3', '\n') + delete('1.3', '2.0') + Equal(get('1.0', '1.end'), 'lloworld') + + def test_delete_slice(self): + delete = self.text.delete + get = self.text.get + Equal = self.assertEqual + self.text.insert('1.0', self.hw) + + delete('1.0', '1.0 lineend') + Equal(get('1.0', 'end'), '\nworld\n') + + delete('1.0', 'end') + Equal(get('1.0', 'end'), '\n') + + self.text.insert('1.0', self.hw) + delete('1.0', '2.0') + Equal(get('1.0', 'end'), 'world\n') + + delete('1.0', 'end') + Equal(get('1.0', 'end'), '\n') + + self.text.insert('1.0', self.hw) + delete('1.2', '2.3') + Equal(get('1.0', 'end'), 'held\n') + + def test_multiple_lines(self): # insert and delete + self.text.insert('1.0', 'hello') + + self.text.insert('1.3', '1\n2\n3\n4\n5') + self.assertEqual(self.text.get('1.0', 'end'), 'hel1\n2\n3\n4\n5lo\n') + + self.text.delete('1.3', '5.1') + self.assertEqual(self.text.get('1.0', 'end'), 'hello\n') + + def test_compare(self): + compare = self.text.compare + Equal = self.assertEqual + # need data so indexes not squished to 1,0 + self.text.insert('1.0', 'First\nSecond\nThird\n') + + self.assertRaises(TclError, compare, '2.2', 'op', '2.2') + + for op, less1, less0, equal, greater0, greater1 in ( + ('<', True, True, False, False, False), + ('<=', True, True, True, False, False), + ('>', False, False, False, True, True), + ('>=', False, False, True, True, True), + ('==', False, False, True, False, False), + ('!=', True, True, False, True, True), + ): + Equal(compare('1.1', op, '2.2'), less1, op) + Equal(compare('2.1', op, '2.2'), less0, op) + Equal(compare('2.2', op, '2.2'), equal, op) + Equal(compare('2.3', op, '2.2'), greater0, op) + Equal(compare('3.3', op, '2.2'), greater1, op) + + +class MockTextTest(TextTest, unittest.TestCase): + + @classmethod + def setUpClass(cls): + from idlelib.idle_test.mock_tk import Text + cls.Text = Text + + def test_decode(self): + # test endflags (-1, 0) not tested by test_index (which uses +1) + decode = self.text._decode + Equal = self.assertEqual + self.text.insert('1.0', self.hw) + + Equal(decode('end', -1), (2, 5)) + Equal(decode('3.1', -1), (2, 5)) + Equal(decode('end', 0), (2, 6)) + Equal(decode('3.1', 0), (2, 6)) + + +class TkTextTest(TextTest, unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + from Tkinter import Tk, Text + cls.Text = Text + cls.root = Tk() + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/test_warning.py b/Lib/idlelib/idle_test/test_warning.py new file mode 100644 index 00000000000..da1d8a1d0ab --- /dev/null +++ b/Lib/idlelib/idle_test/test_warning.py @@ -0,0 +1,73 @@ +'''Test warnings replacement in PyShell.py and run.py. + +This file could be expanded to include traceback overrides +(in same two modules). If so, change name. +Revise if output destination changes (http://bugs.python.org/issue18318). +Make sure warnings module is left unaltered (http://bugs.python.org/issue18081). +''' + +import unittest +from test.test_support import captured_stderr + +import warnings +# Try to capture default showwarning before Idle modules are imported. +showwarning = warnings.showwarning +# But if we run this file within idle, we are in the middle of the run.main loop +# and default showwarnings has already been replaced. +running_in_idle = 'idle' in showwarning.__name__ + +from idlelib import run +from idlelib import PyShell as shell + +# The following was generated from PyShell.idle_formatwarning +# and checked as matching expectation. +idlemsg = ''' +Warning (from warnings module): + File "test_warning.py", line 99 + Line of code +UserWarning: Test +''' +shellmsg = idlemsg + ">>> " + +class RunWarnTest(unittest.TestCase): + + @unittest.skipIf(running_in_idle, "Does not work when run within Idle.") + def test_showwarnings(self): + self.assertIs(warnings.showwarning, showwarning) + run.capture_warnings(True) + self.assertIs(warnings.showwarning, run.idle_showwarning_subproc) + run.capture_warnings(False) + self.assertIs(warnings.showwarning, showwarning) + + def test_run_show(self): + with captured_stderr() as f: + run.idle_showwarning_subproc( + 'Test', UserWarning, 'test_warning.py', 99, f, 'Line of code') + # The following uses .splitlines to erase line-ending differences + self.assertEqual(idlemsg.splitlines(), f.getvalue().splitlines()) + +class ShellWarnTest(unittest.TestCase): + + @unittest.skipIf(running_in_idle, "Does not work when run within Idle.") + def test_showwarnings(self): + self.assertIs(warnings.showwarning, showwarning) + shell.capture_warnings(True) + self.assertIs(warnings.showwarning, shell.idle_showwarning) + shell.capture_warnings(False) + self.assertIs(warnings.showwarning, showwarning) + + def test_idle_formatter(self): + # Will fail if format changed without regenerating idlemsg + s = shell.idle_formatwarning( + 'Test', UserWarning, 'test_warning.py', 99, 'Line of code') + self.assertEqual(idlemsg, s) + + def test_shell_show(self): + with captured_stderr() as f: + shell.idle_showwarning( + 'Test', UserWarning, 'test_warning.py', 99, f, 'Line of code') + self.assertEqual(shellmsg.splitlines(), f.getvalue().splitlines()) + + +if __name__ == '__main__': + unittest.main(verbosity=2, exit=False) diff --git a/Lib/idlelib/idlever.py b/Lib/idlelib/idlever.py index 2aea6e02613..481afab24cb 100644 --- a/Lib/idlelib/idlever.py +++ b/Lib/idlelib/idlever.py @@ -1 +1 @@ -IDLE_VERSION = "2.7.4rc1" +IDLE_VERSION = "2.7.5" diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index e82ebf4c92a..604c5cd54e1 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -22,24 +22,38 @@ LOCALHOST = '127.0.0.1' -try: - import warnings -except ImportError: - pass -else: - def idle_formatwarning_subproc(message, category, filename, lineno, - line=None): - """Format warnings the IDLE way""" - s = "\nWarning (from warnings module):\n" - s += ' File \"%s\", line %s\n' % (filename, lineno) - if line is None: - line = linecache.getline(filename, lineno) - line = line.strip() - if line: - s += " %s\n" % line - s += "%s: %s\n" % (category.__name__, message) - return s - warnings.formatwarning = idle_formatwarning_subproc +import warnings + +def idle_showwarning_subproc( + message, category, filename, lineno, file=None, line=None): + """Show Idle-format warning after replacing warnings.showwarning. + + The only difference is the formatter called. + """ + if file is None: + file = sys.stderr + try: + file.write(PyShell.idle_formatwarning( + message, category, filename, lineno, line)) + except IOError: + pass # the file (probably stderr) is invalid - this warning gets lost. + +_warnings_showwarning = None + +def capture_warnings(capture): + "Replace warning.showwarning with idle_showwarning_subproc, or reverse." + + global _warnings_showwarning + if capture: + if _warnings_showwarning is None: + _warnings_showwarning = warnings.showwarning + warnings.showwarning = idle_showwarning_subproc + else: + if _warnings_showwarning is not None: + warnings.showwarning = _warnings_showwarning + _warnings_showwarning = None + +capture_warnings(True) # Thread shared globals: Establish a queue between a subthread (which handles # the socket) and the main thread (which runs user code), plus global @@ -78,6 +92,8 @@ def main(del_exitfunc=False): except: print>>sys.stderr, "IDLE Subprocess: no IP port passed in sys.argv." return + + capture_warnings(True) sys.argv[:] = [""] sockthread = threading.Thread(target=manage_socket, name='SockThread', @@ -104,6 +120,7 @@ def main(del_exitfunc=False): exit_now = True continue except SystemExit: + capture_warnings(False) raise except: type, value, tb = sys.exc_info() @@ -123,7 +140,7 @@ def manage_socket(address): try: server = MyRPCServer(address, MyHandler) break - except socket.error, err: + except socket.error as err: print>>sys.__stderr__,"IDLE Subprocess: socket error: "\ + err.args[1] + ", retrying...." else: @@ -219,6 +236,7 @@ def exit(): del sys.exitfunc except AttributeError: pass + capture_warnings(False) sys.exit(0) class MyRPCServer(rpc.RPCServer): @@ -264,6 +282,11 @@ def handle(self): IOBinding.encoding) sys.stderr = PyShell.PseudoOutputFile(self.console, "stderr", IOBinding.encoding) + + # Keep a reference to stdin so that it won't try to exit IDLE if + # sys.stdin gets changed from within IDLE's shell. See issue17838. + self._keep_stdin = sys.stdin + self.interp = self.get_remote_proxy("interp") rpc.RPCHandler.getresponse(self, myseq=None, wait=0.05) @@ -347,3 +370,5 @@ def stackviewer(self, flist_oid=None): sys.last_value = val item = StackViewer.StackTreeItem(flist, tb) return RemoteObjectBrowser.remote_object_tree_item(item) + +capture_warnings(False) # Make sure turned off; see issue 18081 diff --git a/Lib/inspect.py b/Lib/inspect.py index 9e928b294f9..933694394dd 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -165,7 +165,7 @@ def isgenerator(object): """Return true if the object is a generator. Generator objects provide these attributes: - __iter__ defined to support interation over container + __iter__ defined to support iteration over container close raises a new GeneratorExit exception inside the generator to terminate the iteration gi_code code object @@ -728,7 +728,8 @@ def getclasstree(classes, unique=0): for parent in c.__bases__: if not parent in children: children[parent] = [] - children[parent].append(c) + if c not in children[parent]: + children[parent].append(c) if unique and parent in classes: break elif c not in roots: roots.append(c) diff --git a/Lib/lib-tk/Tkinter.py b/Lib/lib-tk/Tkinter.py index fcb320f4418..529a3e4b161 100644 --- a/Lib/lib-tk/Tkinter.py +++ b/Lib/lib-tk/Tkinter.py @@ -1347,7 +1347,7 @@ def _grid_configure(self, command, index, cnf, kw): value = words[i+1] if not value: value = None - elif '.' in value: + elif '.' in str(value): value = getdouble(value) else: value = getint(value) @@ -1736,7 +1736,7 @@ def __init__(self, screenName=None, baseName=None, className='Tk', # ensure that self.tk is always _something_. self.tk = None if baseName is None: - import sys, os + import os baseName = os.path.basename(sys.argv[0]) baseName, ext = os.path.splitext(baseName) if ext not in ('.py', '.pyc', '.pyo'): @@ -1880,7 +1880,7 @@ def pack_info(self): for i in range(0, len(words), 2): key = words[i][1:] value = words[i+1] - if value[:1] == '.': + if str(value)[:1] == '.': value = self._nametowidget(value) dict[key] = value return dict @@ -1931,7 +1931,7 @@ def place_info(self): for i in range(0, len(words), 2): key = words[i][1:] value = words[i+1] - if value[:1] == '.': + if str(value)[:1] == '.': value = self._nametowidget(value) dict[key] = value return dict @@ -1980,7 +1980,7 @@ def grid_info(self): for i in range(0, len(words), 2): key = words[i][1:] value = words[i+1] - if value[:1] == '.': + if str(value)[:1] == '.': value = self._nametowidget(value) dict[key] = value return dict diff --git a/Lib/lib-tk/test/test_ttk/test_widgets.py b/Lib/lib-tk/test/test_ttk/test_widgets.py index 9d06a75610e..c594602dd35 100644 --- a/Lib/lib-tk/test/test_ttk/test_widgets.py +++ b/Lib/lib-tk/test/test_ttk/test_widgets.py @@ -26,8 +26,8 @@ def tearDown(self): def test_identify(self): self.widget.update_idletasks() self.assertEqual(self.widget.identify( - int(self.widget.winfo_width() / 2), - int(self.widget.winfo_height() / 2) + self.widget.winfo_width() // 2, + self.widget.winfo_height() // 2 ), "label") self.assertEqual(self.widget.identify(-1, -1), "") @@ -104,7 +104,7 @@ def cb_test(): cbtn['command'] = '' res = cbtn.invoke() - self.assertEqual(res, '') + self.assertEqual(str(res), '') self.assertFalse(len(success) > 1) self.assertEqual(cbtn['offvalue'], cbtn.tk.globalgetvar(cbtn['variable'])) @@ -452,7 +452,7 @@ def cb_test(): cbtn2['command'] = '' res = cbtn2.invoke() - self.assertEqual(res, '') + self.assertEqual(str(res), '') self.assertFalse(len(success) > 1) self.assertEqual(cbtn2['value'], myvar.get()) self.assertEqual(myvar.get(), diff --git a/Lib/lib-tk/turtle.py b/Lib/lib-tk/turtle.py index 29540518b74..f8188e1b707 100644 --- a/Lib/lib-tk/turtle.py +++ b/Lib/lib-tk/turtle.py @@ -1233,7 +1233,7 @@ def delay(self, delay=None): self._delayvalue = int(delay) def _incrementudc(self): - """Increment upadate counter.""" + """Increment update counter.""" if not TurtleScreen._RUNNING: TurtleScreen._RUNNNING = True raise Terminator @@ -2439,7 +2439,7 @@ def __init__(self, canvas=None, self.screen = TurtleScreen(canvas) RawTurtle.screens.append(self.screen) else: - raise TurtleGraphicsError("bad cavas argument %s" % canvas) + raise TurtleGraphicsError("bad canvas argument %s" % canvas) screen = self.screen TNavigator.__init__(self, screen.mode()) @@ -2684,7 +2684,7 @@ def shape(self, name=None): def shapesize(self, stretch_wid=None, stretch_len=None, outline=None): """Set/return turtle's stretchfactors/outline. Set resizemode to "user". - Optinonal arguments: + Optional arguments: stretch_wid : positive number stretch_len : positive number outline : positive number @@ -2975,7 +2975,7 @@ def clearstamps(self, n=None): def _goto(self, end): """Move the pen to the point end, thereby drawing a line - if pen is down. All other methodes for turtle movement depend + if pen is down. All other methods for turtle movement depend on this one. """ ## Version mit undo-stuff diff --git a/Lib/lib2to3/fixes/fix_itertools.py b/Lib/lib2to3/fixes/fix_itertools.py index 27f8a49ec60..067641b8f86 100644 --- a/Lib/lib2to3/fixes/fix_itertools.py +++ b/Lib/lib2to3/fixes/fix_itertools.py @@ -34,8 +34,8 @@ def transform(self, node, results): # Remove the 'itertools' prefix = it.prefix it.remove() - # Replace the node wich contains ('.', 'function') with the - # function (to be consistant with the second part of the pattern) + # Replace the node which contains ('.', 'function') with the + # function (to be consistent with the second part of the pattern) dot.remove() func.parent.replace(func) diff --git a/Lib/lib2to3/fixes/fix_metaclass.py b/Lib/lib2to3/fixes/fix_metaclass.py index c86fbeaab2c..4f5593c5feb 100644 --- a/Lib/lib2to3/fixes/fix_metaclass.py +++ b/Lib/lib2to3/fixes/fix_metaclass.py @@ -71,7 +71,7 @@ def fixup_parse_tree(cls_node): def fixup_simple_stmt(parent, i, stmt_node): """ if there is a semi-colon all the parts count as part of the same simple_stmt. We just want the __metaclass__ part so we move - everything efter the semi-colon into its own simple_stmt node + everything after the semi-colon into its own simple_stmt node """ for semi_ind, node in enumerate(stmt_node.children): if node.type == token.SEMI: # *sigh* diff --git a/Lib/lib2to3/fixes/fix_unicode.py b/Lib/lib2to3/fixes/fix_unicode.py index 6c89576540c..2d776f61051 100644 --- a/Lib/lib2to3/fixes/fix_unicode.py +++ b/Lib/lib2to3/fixes/fix_unicode.py @@ -1,25 +1,42 @@ -"""Fixer that changes unicode to str, unichr to chr, and u"..." into "...". +r"""Fixer for unicode. + +* Changes unicode to str and unichr to chr. + +* If "...\u..." is not unicode literal change it into "...\\u...". + +* Change u"..." into "...". """ -import re from ..pgen2 import token from .. import fixer_base _mapping = {u"unichr" : u"chr", u"unicode" : u"str"} -_literal_re = re.compile(ur"[uU][rR]?[\'\"]") class FixUnicode(fixer_base.BaseFix): BM_compatible = True PATTERN = "STRING | 'unicode' | 'unichr'" + def start_tree(self, tree, filename): + super(FixUnicode, self).start_tree(tree, filename) + self.unicode_literals = 'unicode_literals' in tree.future_features + def transform(self, node, results): if node.type == token.NAME: new = node.clone() new.value = _mapping[node.value] return new elif node.type == token.STRING: - if _literal_re.match(node.value): - new = node.clone() - new.value = new.value[1:] - return new + val = node.value + if not self.unicode_literals and val[0] in u'\'"' and u'\\' in val: + val = ur'\\'.join([ + v.replace(u'\\u', ur'\\u').replace(u'\\U', ur'\\U') + for v in val.split(ur'\\') + ]) + if val[0] in u'uU': + val = val[1:] + if val == node.value: + return node + new = node.clone() + new.value = val + return new diff --git a/Lib/lib2to3/pgen2/tokenize.py b/Lib/lib2to3/pgen2/tokenize.py index e090aa98ddb..f6e0284c2f5 100644 --- a/Lib/lib2to3/pgen2/tokenize.py +++ b/Lib/lib2to3/pgen2/tokenize.py @@ -236,7 +236,7 @@ def compat(self, token, iterable): startline = False toks_append(tokval) -cookie_re = re.compile("coding[:=]\s*([-\w.]+)") +cookie_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') def _get_normal_name(orig_enc): """Imitates get_normal_name in tokenizer.c.""" @@ -281,11 +281,10 @@ def find_cookie(line): line_string = line.decode('ascii') except UnicodeDecodeError: return None - - matches = cookie_re.findall(line_string) - if not matches: + match = cookie_re.match(line_string) + if not match: return None - encoding = _get_normal_name(matches[0]) + encoding = _get_normal_name(match.group(1)) try: codec = lookup(encoding) except LookupError: diff --git a/Lib/lib2to3/tests/data/false_encoding.py b/Lib/lib2to3/tests/data/false_encoding.py new file mode 100644 index 00000000000..f4e59e787da --- /dev/null +++ b/Lib/lib2to3/tests/data/false_encoding.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python +print '#coding=0' diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index 15482812201..6801c1703bb 100644 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -1405,27 +1405,27 @@ def test_27(self): a = "d.values()" self.check(b, a) - def test_14(self): + def test_28(self): b = "[i for i in d.viewkeys()]" a = "[i for i in d.keys()]" self.check(b, a) - def test_15(self): + def test_29(self): b = "(i for i in d.viewkeys())" a = "(i for i in d.keys())" self.check(b, a) - def test_17(self): + def test_30(self): b = "iter(d.viewkeys())" a = "iter(d.keys())" self.check(b, a) - def test_18(self): + def test_31(self): b = "list(d.viewkeys())" a = "list(d.keys())" self.check(b, a) - def test_19(self): + def test_32(self): b = "sorted(d.viewkeys())" a = "sorted(d.keys())" self.check(b, a) @@ -2824,6 +2824,43 @@ def test_unicode_literal_3(self): a = """R'''x''' """ self.check(b, a) + def test_native_literal_escape_u(self): + b = """'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + a = """'\\\\\\\\u20ac\\\\U0001d121\\\\u20ac'""" + self.check(b, a) + + b = """r'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + a = """r'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + self.check(b, a) + + def test_bytes_literal_escape_u(self): + b = """b'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + a = """b'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + self.check(b, a) + + b = """br'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + a = """br'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + self.check(b, a) + + def test_unicode_literal_escape_u(self): + b = """u'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + a = """'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + self.check(b, a) + + b = """ur'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + a = """r'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + self.check(b, a) + + def test_native_unicode_literal_escape_u(self): + f = 'from __future__ import unicode_literals\n' + b = f + """'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + a = f + """'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + self.check(b, a) + + b = f + """r'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + a = f + """r'\\\\\\u20ac\\U0001d121\\\\u20ac'""" + self.check(b, a) + class Test_callable(FixerTestCase): fixer = "callable" diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index 26023813516..f66cb6b3f87 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -73,7 +73,7 @@ def test_3x_style_invalid_4(self): self.invalid_syntax("raise E from") -# Adapated from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef +# Adaptated from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef class TestFunctionAnnotations(GrammarTest): def test_1(self): self.validate("""def f(x) -> list: pass""") diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py index 6020d1ff18b..543fd421366 100644 --- a/Lib/lib2to3/tests/test_refactor.py +++ b/Lib/lib2to3/tests/test_refactor.py @@ -271,6 +271,10 @@ def test_file_encoding(self): fn = os.path.join(TEST_DATA_DIR, "different_encoding.py") self.check_file_refactoring(fn) + def test_false_file_encoding(self): + fn = os.path.join(TEST_DATA_DIR, "false_encoding.py") + data = self.check_file_refactoring(fn) + def test_bom(self): fn = os.path.join(TEST_DATA_DIR, "bom.py") data = self.check_file_refactoring(fn) diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 1a622a3d074..0e8e3ddde0d 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -893,6 +893,7 @@ def __init__(self, filename, mode='a', encoding=None, delay=0): self.baseFilename = os.path.abspath(filename) self.mode = mode self.encoding = encoding + self.delay = delay if delay: #We don't open the stream, but we still need to call the #Handler constructor to set level, formatter, lock etc. diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 26dfe4a1d18..c45a3132ba7 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -1,4 +1,4 @@ -# Copyright 2001-2012 by Vinay Sajip. All Rights Reserved. +# Copyright 2001-2013 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, @@ -18,7 +18,7 @@ Additional handlers for the logging package for Python. The core package is based on PEP 282 and comments thereto in comp.lang.python. -Copyright (C) 2001-2012 Vinay Sajip. All Rights Reserved. +Copyright (C) 2001-2013 Vinay Sajip. All Rights Reserved. To use, simply 'import logging.handlers' and log away! """ @@ -137,9 +137,11 @@ def doRollover(self): dfn = self.baseFilename + ".1" if os.path.exists(dfn): os.remove(dfn) - os.rename(self.baseFilename, dfn) - #print "%s -> %s" % (self.baseFilename, dfn) - self.stream = self._open() + # Issue 18940: A file may not have been created if delay is True. + if os.path.exists(self.baseFilename): + os.rename(self.baseFilename, dfn) + if not self.delay: + self.stream = self._open() def shouldRollover(self, record): """ @@ -343,17 +345,14 @@ def doRollover(self): dfn = self.baseFilename + "." + time.strftime(self.suffix, timeTuple) if os.path.exists(dfn): os.remove(dfn) - os.rename(self.baseFilename, dfn) + # Issue 18940: A file may not have been created if delay is True. + if os.path.exists(self.baseFilename): + os.rename(self.baseFilename, dfn) if self.backupCount > 0: - # find the oldest log file and delete it - #s = glob.glob(self.baseFilename + ".20*") - #if len(s) > self.backupCount: - # s.sort() - # os.remove(s[0]) for s in self.getFilesToDelete(): os.remove(s) - #print "%s -> %s" % (self.baseFilename, dfn) - self.stream = self._open() + if not self.delay: + self.stream = self._open() newRolloverAt = self.computeRollover(currentTime) while newRolloverAt <= currentTime: newRolloverAt = newRolloverAt + self.interval @@ -737,13 +736,17 @@ class SysLogHandler(logging.Handler): } def __init__(self, address=('localhost', SYSLOG_UDP_PORT), - facility=LOG_USER, socktype=socket.SOCK_DGRAM): + facility=LOG_USER, socktype=None): """ Initialize a handler. If address is specified as a string, a UNIX socket is used. To log to a local syslogd, "SysLogHandler(address="/dev/log")" can be used. - If facility is not specified, LOG_USER is used. + If facility is not specified, LOG_USER is used. If socktype is + specified as socket.SOCK_DGRAM or socket.SOCK_STREAM, that specific + socket type will be used. For Unix sockets, you can also specify a + socktype of None, in which case socket.SOCK_DGRAM will be used, falling + back to socket.SOCK_STREAM. """ logging.Handler.__init__(self) @@ -756,18 +759,37 @@ def __init__(self, address=('localhost', SYSLOG_UDP_PORT), self._connect_unixsocket(address) else: self.unixsocket = 0 + if socktype is None: + socktype = socket.SOCK_DGRAM self.socket = socket.socket(socket.AF_INET, socktype) if socktype == socket.SOCK_STREAM: self.socket.connect(address) + self.socktype = socktype self.formatter = None def _connect_unixsocket(self, address): - self.socket = socket.socket(socket.AF_UNIX, self.socktype) + use_socktype = self.socktype + if use_socktype is None: + use_socktype = socket.SOCK_DGRAM + self.socket = socket.socket(socket.AF_UNIX, use_socktype) try: self.socket.connect(address) + # it worked, so set self.socktype to the used type + self.socktype = use_socktype except socket.error: self.socket.close() - raise + if self.socktype is not None: + # user didn't specify falling back, so fail + raise + use_socktype = socket.SOCK_STREAM + self.socket = socket.socket(socket.AF_UNIX, use_socktype) + try: + self.socket.connect(address) + # it worked, so set self.socktype to the used type + self.socktype = use_socktype + except socket.error: + self.socket.close() + raise # curious: when talking to the unix-domain '/dev/log' socket, a # zero-terminator seems to be required. this string is placed @@ -833,6 +855,7 @@ def emit(self, record): try: self.socket.send(msg) except socket.error: + self.socket.close() # See issue 17981 self._connect_unixsocket(self.address) self.socket.send(msg) elif self.socktype == socket.SOCK_DGRAM: diff --git a/Lib/mailbox.py b/Lib/mailbox.py index 530d3c5a66e..4cdd7d45ea1 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -286,6 +286,12 @@ def add(self, message): suffix = '' uniq = os.path.basename(tmp_file.name).split(self.colon)[0] dest = os.path.join(self._path, subdir, uniq + suffix) + if isinstance(message, MaildirMessage): + os.utime(tmp_file.name, + (os.path.getatime(tmp_file.name), message.get_date())) + # No file modification should be done after the file is moved to its + # final position in order to prevent race conditions with changes + # from other programs try: if hasattr(os, 'link'): os.link(tmp_file.name, dest) @@ -299,8 +305,6 @@ def add(self, message): % dest) else: raise - if isinstance(message, MaildirMessage): - os.utime(dest, (os.path.getatime(dest), message.get_date())) return uniq def remove(self, key): @@ -335,11 +339,15 @@ def __setitem__(self, key, message): else: suffix = '' self.discard(key) + tmp_path = os.path.join(self._path, temp_subpath) new_path = os.path.join(self._path, subdir, key + suffix) - os.rename(os.path.join(self._path, temp_subpath), new_path) if isinstance(message, MaildirMessage): - os.utime(new_path, (os.path.getatime(new_path), - message.get_date())) + os.utime(tmp_path, + (os.path.getatime(tmp_path), message.get_date())) + # No file modification should be done after the file is moved to its + # final position in order to prevent race conditions with changes + # from other programs + os.rename(tmp_path, new_path) def get_message(self, key): """Return a Message representation or raise a KeyError.""" diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index ba4ea8257b6..18ade7381ad 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -386,12 +386,14 @@ def _default_mime_types(): '.taz': '.tar.gz', '.tz': '.tar.gz', '.tbz2': '.tar.bz2', + '.txz': '.tar.xz', } encodings_map = { '.gz': 'gzip', '.Z': 'compress', '.bz2': 'bzip2', + '.xz': 'xz', } # Before adding new types, make sure they are either registered with IANA, diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index 7f2bf8b1588..f6f84f3fdea 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -516,7 +516,7 @@ def report(self): # Print modules that may be missing, but then again, maybe not... if maybe: print - print "Submodules thay appear to be missing, but could also be", + print "Submodules that appear to be missing, but could also be", print "global names in the parent package:" for name in maybe: mods = self.badmodules[name].keys() diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 57bf811eda5..1a29c36f639 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -270,7 +270,14 @@ def __init__(self, address, family, backlog=1): self._unlink = None def accept(self): - s, self._last_accepted = self._socket.accept() + while True: + try: + s, self._last_accepted = self._socket.accept() + except socket.error as e: + if e.args[0] != errno.EINTR: + raise + else: + break s.setblocking(True) fd = duplicate(s.fileno()) conn = _multiprocessing.Connection(fd) @@ -287,15 +294,16 @@ def SocketClient(address): ''' Return a connection object connected to the socket given by `address` ''' - family = address_type(address) - s = socket.socket( getattr(socket, family) ) - s.setblocking(True) + family = getattr(socket, address_type(address)) t = _init_timeout() while 1: + s = socket.socket(family) + s.setblocking(True) try: s.connect(address) except socket.error, e: + s.close() if e.args[0] != errno.ECONNREFUSED or _check_timeout(t): debug('failed to connect to address %s', address) raise diff --git a/Lib/multiprocessing/forking.py b/Lib/multiprocessing/forking.py index dc465b4ed25..6bddfb74ff5 100644 --- a/Lib/multiprocessing/forking.py +++ b/Lib/multiprocessing/forking.py @@ -367,7 +367,7 @@ def get_command_line(): def main(): ''' - Run code specifed by data received over pipe + Run code specified by data received over pipe ''' assert is_forking(sys.argv) diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index ffe5812b89d..08d35d86828 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -763,6 +763,7 @@ def _callmethod(self, methodname, args=(), kwds={}): elif kind == '#PROXY': exposed, token = result proxytype = self._manager._registry[token.typeid][-1] + token.address = self._token.address proxy = proxytype( token, self._serializer, manager=self._manager, authkey=self._authkey, exposed=exposed diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 00c904a8a73..4d1860029c1 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -565,6 +565,8 @@ def _set(self, i, obj): self._cond.release() del self._cache[self._job] +AsyncResult = ApplyResult # create alias -- see #17805 + # # Class whose instances are returned by `Pool.map_async()` # diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 4b077e53aed..d845f72a1c4 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -226,7 +226,7 @@ def __repr__(self): num_waiters = (self._sleeping_count._semlock._get_value() - self._woken_count._semlock._get_value()) except Exception: - num_waiters = 'unkown' + num_waiters = 'unknown' return '' % (self._lock, num_waiters) def wait(self, timeout=None): diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 3ce480fc420..d1b3d2e3e87 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -329,10 +329,13 @@ def _exit_function(info=info, debug=debug, _run_finalizers=_run_finalizers, class ForkAwareThreadLock(object): def __init__(self): + self._reset() + register_after_fork(self, ForkAwareThreadLock._reset) + + def _reset(self): self._lock = threading.Lock() self.acquire = self._lock.acquire self.release = self._lock.release - register_after_fork(self, ForkAwareThreadLock.__init__) class ForkAwareLocal(threading.local): def __init__(self): diff --git a/Lib/netrc.py b/Lib/netrc.py index 0fd37e304f5..4b18973d51e 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -2,7 +2,9 @@ # Module and documentation by Eric S. Raymond, 21 Dec 1998 -import os, shlex +import os, stat, shlex +if os.name == 'posix': + import pwd __all__ = ["netrc", "NetrcParseError"] @@ -21,6 +23,7 @@ def __str__(self): class netrc: def __init__(self, file=None): + default_netrc = file is None if file is None: try: file = os.path.join(os.environ['HOME'], ".netrc") @@ -29,9 +32,9 @@ def __init__(self, file=None): self.hosts = {} self.macros = {} with open(file) as fp: - self._parse(file, fp) + self._parse(file, fp, default_netrc) - def _parse(self, file, fp): + def _parse(self, file, fp, default_netrc): lexer = shlex.shlex(fp) lexer.wordchars += r"""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~""" lexer.commenters = lexer.commenters.replace('#', '') @@ -88,6 +91,26 @@ def _parse(self, file, fp): elif tt == 'account': account = lexer.get_token() elif tt == 'password': + if os.name == 'posix' and default_netrc: + prop = os.fstat(fp.fileno()) + if prop.st_uid != os.getuid(): + try: + fowner = pwd.getpwuid(prop.st_uid)[0] + except KeyError: + fowner = 'uid %s' % prop.st_uid + try: + user = pwd.getpwuid(os.getuid())[0] + except KeyError: + user = 'uid %s' % os.getuid() + raise NetrcParseError( + ("~/.netrc file owner (%s) does not match" + " current user (%s)") % (fowner, user), + file, lexer.lineno) + if (prop.st_mode & (stat.S_IRWXG | stat.S_IRWXO)): + raise NetrcParseError( + "~/.netrc access too permissive: access" + " permissions must restrict access to only" + " the owner", file, lexer.lineno) password = lexer.get_token() else: raise NetrcParseError("bad follower token %r" % tt, diff --git a/Lib/nntplib.py b/Lib/nntplib.py index 2dc82a90ae1..81ebe4bc035 100644 --- a/Lib/nntplib.py +++ b/Lib/nntplib.py @@ -37,6 +37,13 @@ "error_reply","error_temp","error_perm","error_proto", "error_data",] +# maximal line length when calling readline(). This is to prevent +# reading arbitrary length lines. RFC 3977 limits NNTP line length to +# 512 characters, including CRLF. We have selected 2048 just to be on +# the safe side. +_MAXLINE = 2048 + + # Exceptions raised when an error or invalid response is received class NNTPError(Exception): """Base class for all nntplib exceptions""" @@ -200,7 +207,9 @@ def putcmd(self, line): def getline(self): """Internal: return one line from the server, stripping CRLF. Raise EOFError if the connection is closed.""" - line = self.file.readline() + line = self.file.readline(_MAXLINE + 1) + if len(line) > _MAXLINE: + raise NNTPDataError('line too long') if self.debugging > 1: print '*get*', repr(line) if not line: raise EOFError diff --git a/Lib/optparse.py b/Lib/optparse.py index 731a2bb5edf..6c3f45651ff 100644 --- a/Lib/optparse.py +++ b/Lib/optparse.py @@ -1471,7 +1471,7 @@ def _match_long_opt(self, opt): """_match_long_opt(opt : string) -> string Determine which long option string 'opt' matches, ie. which one - it is an unambiguous abbrevation for. Raises BadOptionError if + it is an unambiguous abbreviation for. Raises BadOptionError if 'opt' doesn't unambiguously match any long option string. """ return _match_abbrev(opt, self._long_opt) diff --git a/Lib/pdb.py b/Lib/pdb.py index 5468d3fcaff..113b4e08927 100755 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -1095,7 +1095,7 @@ def help_exec(self): def help_run(self): print """run [args...] Restart the debugged python program. If a string is supplied, it is -splitted with "shlex" and the result is used as the new sys.argv. +split with "shlex" and the result is used as the new sys.argv. History, breakpoints, actions and debugger options are preserved. "restart" is an alias for "run".""" diff --git a/Lib/pickle.py b/Lib/pickle.py index 5b95cbaca76..508e858d816 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -962,7 +962,7 @@ def load_string(self): rep = self.readline()[:-1] for q in "\"'": # double or single quote if rep.startswith(q): - if not rep.endswith(q): + if len(rep) < 2 or not rep.endswith(q): raise ValueError, "insecure string pickle" rep = rep[len(q):-len(q)] break diff --git a/Lib/pickletools.py b/Lib/pickletools.py index d717728d417..8de53dd2507 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -804,7 +804,7 @@ def __repr__(self): obtype=StackObject, doc="""An object representing a contiguous slice of the stack. - This is used in conjuction with markobject, to represent all + This is used in conjunction with markobject, to represent all of the stack following the topmost markobject. For example, the POP_MARK opcode changes the stack from @@ -1929,7 +1929,7 @@ def dis(pickle, out=None, memo=None, indentlevel=4): stack = [] # crude emulation of unpickler stack if memo is None: - memo = {} # crude emulation of unpicker memo + memo = {} # crude emulation of unpickler memo maxproto = -1 # max protocol number seen markstack = [] # bytecode positions of MARK opcodes indentchunk = ' ' * indentlevel diff --git a/Lib/plat-mac/EasyDialogs.py b/Lib/plat-mac/EasyDialogs.py index 129cf2cae26..1d3edb3aacd 100644 --- a/Lib/plat-mac/EasyDialogs.py +++ b/Lib/plat-mac/EasyDialogs.py @@ -243,8 +243,15 @@ def AskYesNoCancel(question, default = 0, yes=None, no=None, cancel=None, id=262 +# The deprecated Carbon QuickDraw APIs are no longer available as of +# OS X 10.8. Raise an ImportError here in that case so that callers +# of EasyDialogs, like BuildApplet, will do the right thing. + +try: + screenbounds = Qd.GetQDGlobalsScreenBits().bounds +except AttributeError: + raise ImportError("QuickDraw APIs not available") -screenbounds = Qd.GetQDGlobalsScreenBits().bounds screenbounds = screenbounds[0]+4, screenbounds[1]+4, \ screenbounds[2]-4, screenbounds[3]-4 diff --git a/Lib/platform.py b/Lib/platform.py index c0016a8198c..b6dab3b2349 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -228,7 +228,7 @@ def _dist_try_harder(distname,version,id): return 'OpenLinux',pkg[1],id if os.path.isdir('/usr/lib/setup'): - # Check for slackware verson tag file (thanks to Greg Andruk) + # Check for slackware version tag file (thanks to Greg Andruk) verfiles = os.listdir('/usr/lib/setup') for n in range(len(verfiles)-1, -1, -1): if verfiles[n][:14] != 'slack-version-': @@ -280,7 +280,7 @@ def _parse_release_file(firstline): if m is not None: return tuple(m.groups()) - # Unkown format... take the first two words + # Unknown format... take the first two words l = string.split(string.strip(firstline)) if l: version = l[0] @@ -800,7 +800,7 @@ def mac_ver(release='',versioninfo=('','',''),machine=''): versioninfo, machine) with versioninfo being a tuple (version, dev_stage, non_release_version). - Entries which cannot be determined are set to the paramter values + Entries which cannot be determined are set to the parameter values which default to ''. All tuple entries are strings. """ diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 51944eecb0f..42897b8da8b 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -262,8 +262,8 @@ def writeValue(self, value): def writeData(self, data): self.beginElement("data") self.indentLevel -= 1 - maxlinelength = 76 - len(self.indent.replace("\t", " " * 8) * - self.indentLevel) + maxlinelength = max(16, 76 - len(self.indent.replace("\t", " " * 8) * + self.indentLevel)) for line in data.asBase64(maxlinelength).split("\n"): if line: self.writeln(line) diff --git a/Lib/poplib.py b/Lib/poplib.py index e2b33ef10f4..dc7cbdf062e 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -321,7 +321,7 @@ class POP3_SSL(POP3): hostname - the hostname of the pop3 over ssl server port - port number - keyfile - PEM formatted file that countains your private key + keyfile - PEM formatted file that contains your private key certfile - PEM formatted certificate chain file See the methods of the parent class POP3 for more documentation. diff --git a/Lib/pprint.py b/Lib/pprint.py index 910283e6091..77f2a566f35 100644 --- a/Lib/pprint.py +++ b/Lib/pprint.py @@ -37,7 +37,10 @@ import sys as _sys import warnings -from cStringIO import StringIO as _StringIO +try: + from cStringIO import StringIO as _StringIO +except ImportError: + from StringIO import StringIO as _StringIO __all__ = ["pprint","pformat","isreadable","isrecursive","saferepr", "PrettyPrinter"] @@ -182,25 +185,18 @@ def _format(self, object, stream, indent, allowance, context, level): if issubclass(typ, list): write('[') endchar = ']' - elif issubclass(typ, set): - if not length: - write('set()') - return - write('set([') - endchar = '])' - object = _sorted(object) - indent += 4 - elif issubclass(typ, frozenset): + elif issubclass(typ, tuple): + write('(') + endchar = ')' + else: if not length: - write('frozenset()') + write(rep) return - write('frozenset([') + write(typ.__name__) + write('([') endchar = '])' + indent += len(typ.__name__) + 1 object = _sorted(object) - indent += 10 - else: - write('(') - endchar = ')' if self._indent_per_level > 1 and sepLines: write((self._indent_per_level - 1) * ' ') if length: diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 63926d38bc1..eb1c15e7180 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,7 +1,7 @@ -# Autogenerated by Sphinx on Sat Mar 23 10:09:08 2013 +# Autogenerated by Sphinx on Sat May 11 22:31:13 2013 topics = {'assert': '\nThe ``assert`` statement\n************************\n\nAssert statements are a convenient way to insert debugging assertions\ninto a program:\n\n assert_stmt ::= "assert" expression ["," expression]\n\nThe simple form, ``assert expression``, is equivalent to\n\n if __debug__:\n if not expression: raise AssertionError\n\nThe extended form, ``assert expression1, expression2``, is equivalent\nto\n\n if __debug__:\n if not expression1: raise AssertionError(expression2)\n\nThese equivalences assume that ``__debug__`` and ``AssertionError``\nrefer to the built-in variables with those names. In the current\nimplementation, the built-in variable ``__debug__`` is ``True`` under\nnormal circumstances, ``False`` when optimization is requested\n(command line option -O). The current code generator emits no code\nfor an assert statement when optimization is requested at compile\ntime. Note that it is unnecessary to include the source code for the\nexpression that failed in the error message; it will be displayed as\npart of the stack trace.\n\nAssignments to ``__debug__`` are illegal. The value for the built-in\nvariable is determined when the interpreter starts.\n', 'assignment': '\nAssignment statements\n*********************\n\nAssignment statements are used to (re)bind names to values and to\nmodify attributes or items of mutable objects:\n\n assignment_stmt ::= (target_list "=")+ (expression_list | yield_expression)\n target_list ::= target ("," target)* [","]\n target ::= identifier\n | "(" target_list ")"\n | "[" target_list "]"\n | attributeref\n | subscription\n | slicing\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn assignment statement evaluates the expression list (remember that\nthis can be a single expression or a comma-separated list, the latter\nyielding a tuple) and assigns the single resulting object to each of\nthe target lists, from left to right.\n\nAssignment is defined recursively depending on the form of the target\n(list). When a target is part of a mutable object (an attribute\nreference, subscription or slicing), the mutable object must\nultimately perform the assignment and decide about its validity, and\nmay raise an exception if the assignment is unacceptable. The rules\nobserved by various types and the exceptions raised are given with the\ndefinition of the object types (see section *The standard type\nhierarchy*).\n\nAssignment of an object to a target list is recursively defined as\nfollows.\n\n* If the target list is a single target: The object is assigned to\n that target.\n\n* If the target list is a comma-separated list of targets: The object\n must be an iterable with the same number of items as there are\n targets in the target list, and the items are assigned, from left to\n right, to the corresponding targets.\n\nAssignment of an object to a single target is recursively defined as\nfollows.\n\n* If the target is an identifier (name):\n\n * If the name does not occur in a ``global`` statement in the\n current code block: the name is bound to the object in the current\n local namespace.\n\n * Otherwise: the name is bound to the object in the current global\n namespace.\n\n The name is rebound if it was already bound. This may cause the\n reference count for the object previously bound to the name to reach\n zero, causing the object to be deallocated and its destructor (if it\n has one) to be called.\n\n* If the target is a target list enclosed in parentheses or in square\n brackets: The object must be an iterable with the same number of\n items as there are targets in the target list, and its items are\n assigned, from left to right, to the corresponding targets.\n\n* If the target is an attribute reference: The primary expression in\n the reference is evaluated. It should yield an object with\n assignable attributes; if this is not the case, ``TypeError`` is\n raised. That object is then asked to assign the assigned object to\n the given attribute; if it cannot perform the assignment, it raises\n an exception (usually but not necessarily ``AttributeError``).\n\n Note: If the object is a class instance and the attribute reference\n occurs on both sides of the assignment operator, the RHS expression,\n ``a.x`` can access either an instance attribute or (if no instance\n attribute exists) a class attribute. The LHS target ``a.x`` is\n always set as an instance attribute, creating it if necessary.\n Thus, the two occurrences of ``a.x`` do not necessarily refer to the\n same attribute: if the RHS expression refers to a class attribute,\n the LHS creates a new instance attribute as the target of the\n assignment:\n\n class Cls:\n x = 3 # class variable\n inst = Cls()\n inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3\n\n This description does not necessarily apply to descriptor\n attributes, such as properties created with ``property()``.\n\n* If the target is a subscription: The primary expression in the\n reference is evaluated. It should yield either a mutable sequence\n object (such as a list) or a mapping object (such as a dictionary).\n Next, the subscript expression is evaluated.\n\n If the primary is a mutable sequence object (such as a list), the\n subscript must yield a plain integer. If it is negative, the\n sequence\'s length is added to it. The resulting value must be a\n nonnegative integer less than the sequence\'s length, and the\n sequence is asked to assign the assigned object to its item with\n that index. If the index is out of range, ``IndexError`` is raised\n (assignment to a subscripted sequence cannot add new items to a\n list).\n\n If the primary is a mapping object (such as a dictionary), the\n subscript must have a type compatible with the mapping\'s key type,\n and the mapping is then asked to create a key/datum pair which maps\n the subscript to the assigned object. This can either replace an\n existing key/value pair with the same key value, or insert a new\n key/value pair (if no key with the same value existed).\n\n* If the target is a slicing: The primary expression in the reference\n is evaluated. It should yield a mutable sequence object (such as a\n list). The assigned object should be a sequence object of the same\n type. Next, the lower and upper bound expressions are evaluated,\n insofar they are present; defaults are zero and the sequence\'s\n length. The bounds should evaluate to (small) integers. If either\n bound is negative, the sequence\'s length is added to it. The\n resulting bounds are clipped to lie between zero and the sequence\'s\n length, inclusive. Finally, the sequence object is asked to replace\n the slice with the items of the assigned sequence. The length of\n the slice may be different from the length of the assigned sequence,\n thus changing the length of the target sequence, if the object\n allows it.\n\n**CPython implementation detail:** In the current implementation, the\nsyntax for targets is taken to be the same as for expressions, and\ninvalid syntax is rejected during the code generation phase, causing\nless detailed error messages.\n\nWARNING: Although the definition of assignment implies that overlaps\nbetween the left-hand side and the right-hand side are \'safe\' (for\nexample ``a, b = b, a`` swaps two variables), overlaps *within* the\ncollection of assigned-to variables are not safe! For instance, the\nfollowing program prints ``[0, 2]``:\n\n x = [0, 1]\n i = 0\n i, x[i] = 1, 2\n print x\n\n\nAugmented assignment statements\n===============================\n\nAugmented assignment is the combination, in a single statement, of a\nbinary operation and an assignment statement:\n\n augmented_assignment_stmt ::= augtarget augop (expression_list | yield_expression)\n augtarget ::= identifier | attributeref | subscription | slicing\n augop ::= "+=" | "-=" | "*=" | "/=" | "//=" | "%=" | "**="\n | ">>=" | "<<=" | "&=" | "^=" | "|="\n\n(See section *Primaries* for the syntax definitions for the last three\nsymbols.)\n\nAn augmented assignment evaluates the target (which, unlike normal\nassignment statements, cannot be an unpacking) and the expression\nlist, performs the binary operation specific to the type of assignment\non the two operands, and assigns the result to the original target.\nThe target is only evaluated once.\n\nAn augmented assignment expression like ``x += 1`` can be rewritten as\n``x = x + 1`` to achieve a similar, but not exactly equal effect. In\nthe augmented version, ``x`` is only evaluated once. Also, when\npossible, the actual operation is performed *in-place*, meaning that\nrather than creating a new object and assigning that to the target,\nthe old object is modified instead.\n\nWith the exception of assigning to tuples and multiple targets in a\nsingle statement, the assignment done by augmented assignment\nstatements is handled the same way as normal assignments. Similarly,\nwith the exception of the possible *in-place* behavior, the binary\noperation performed by augmented assignment is the same as the normal\nbinary operations.\n\nFor targets which are attribute references, the same *caveat about\nclass and instance attributes* applies as for regular assignments.\n', - 'atom-identifiers': '\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name. See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a ``NameError`` exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them. The transformation inserts the\nclass name in front of the name, with leading underscores removed, and\na single underscore inserted in front of the class name. For example,\nthe identifier ``__spam`` occurring in a class named ``Ham`` will be\ntransformed to ``_Ham__spam``. This transformation is independent of\nthe syntactical context in which the identifier is used. If the\ntransformed name is extremely long (longer than 255 characters),\nimplementation defined truncation may happen. If the class name\nconsists only of underscores, no transformation is done.\n', + 'atom-identifiers': '\nIdentifiers (Names)\n*******************\n\nAn identifier occurring as an atom is a name. See section\n*Identifiers and keywords* for lexical definition and section *Naming\nand binding* for documentation of naming and binding.\n\nWhen the name is bound to an object, evaluation of the atom yields\nthat object. When a name is not bound, an attempt to evaluate it\nraises a ``NameError`` exception.\n\n**Private name mangling:** When an identifier that textually occurs in\na class definition begins with two or more underscore characters and\ndoes not end in two or more underscores, it is considered a *private\nname* of that class. Private names are transformed to a longer form\nbefore code is generated for them. The transformation inserts the\nclass name, with leading underscores removed and a single underscore\ninserted, in front of the name. For example, the identifier\n``__spam`` occurring in a class named ``Ham`` will be transformed to\n``_Ham__spam``. This transformation is independent of the syntactical\ncontext in which the identifier is used. If the transformed name is\nextremely long (longer than 255 characters), implementation defined\ntruncation may happen. If the class name consists only of underscores,\nno transformation is done.\n', 'atom-literals': "\nLiterals\n********\n\nPython supports string literals and various numeric literals:\n\n literal ::= stringliteral | integer | longinteger\n | floatnumber | imagnumber\n\nEvaluation of a literal yields an object of the given type (string,\ninteger, long integer, floating point number, complex number) with the\ngiven value. The value may be approximated in the case of floating\npoint and imaginary (complex) literals. See section *Literals* for\ndetails.\n\nAll literals correspond to immutable data types, and hence the\nobject's identity is less important than its value. Multiple\nevaluations of literals with the same value (either the same\noccurrence in the program text or a different occurrence) may obtain\nthe same object or a different object with the same value.\n", 'attribute-access': '\nCustomizing attribute access\n****************************\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should not simply execute ``self.name = value`` --- this would\n cause a recursive call to itself. Instead, it should insert the\n value in the dictionary of instance attributes, e.g.,\n ``self.__dict__[name] = value``. For new-style classes, rather\n than accessing the instance dictionary, it should call the base\n class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\n\nMore attribute access for new-style classes\n===========================================\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup for new-style\n classes*.\n\n\nImplementing Descriptors\n========================\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents). In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n====================\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called. Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to a new-style object instance, ``a.x`` is transformed\n into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a new-style class, ``A.x`` is transformed into the\n call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n=========\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage. This wastes space for objects\nhaving very few instance variables. The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition. The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable. Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n new-style class, *__slots__* reserves space for the declared\n variables and prevents the automatic creation of *__dict__* and\n *__weakref__* for each instance.\n\n New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n *__slots__* declaration would not enable the assignment of new\n attributes not specifically listed in the sequence of instance\n variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n *__slots__* declaration would not enable support for weak\n references.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``long``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n Changed in version 2.6: Previously, *__class__* assignment raised an\n error if either new or old class had *__slots__*.\n', 'attribute-references': '\nAttribute references\n********************\n\nAn attribute reference is a primary followed by a period and a name:\n\n attributeref ::= primary "." identifier\n\nThe primary must evaluate to an object of a type that supports\nattribute references, e.g., a module, list, or an instance. This\nobject is then asked to produce the attribute whose name is the\nidentifier. If this attribute is not available, the exception\n``AttributeError`` is raised. Otherwise, the type and value of the\nobject produced is determined by the object. Multiple evaluations of\nthe same attribute reference may yield different objects.\n', @@ -33,14 +33,14 @@ 'exprlists': '\nExpression lists\n****************\n\n expression_list ::= expression ( "," expression )* [","]\n\nAn expression list containing at least one comma yields a tuple. The\nlength of the tuple is the number of expressions in the list. The\nexpressions are evaluated from left to right.\n\nThe trailing comma is required only to create a single tuple (a.k.a. a\n*singleton*); it is optional in all other cases. A single expression\nwithout a trailing comma doesn\'t create a tuple, but rather yields the\nvalue of that expression. (To create an empty tuple, use an empty pair\nof parentheses: ``()``.)\n', 'floating': '\nFloating point literals\n***********************\n\nFloating point literals are described by the following lexical\ndefinitions:\n\n floatnumber ::= pointfloat | exponentfloat\n pointfloat ::= [intpart] fraction | intpart "."\n exponentfloat ::= (intpart | pointfloat) exponent\n intpart ::= digit+\n fraction ::= "." digit+\n exponent ::= ("e" | "E") ["+" | "-"] digit+\n\nNote that the integer and exponent parts of floating point numbers can\nlook like octal integers, but are interpreted using radix 10. For\nexample, ``077e010`` is legal, and denotes the same number as\n``77e10``. The allowed range of floating point literals is\nimplementation-dependent. Some examples of floating point literals:\n\n 3.14 10. .001 1e100 3.14e-10 0e0\n\nNote that numeric literals do not include a sign; a phrase like ``-1``\nis actually an expression composed of the unary operator ``-`` and the\nliteral ``1``.\n', 'for': '\nThe ``for`` statement\n*********************\n\nThe ``for`` statement is used to iterate over the elements of a\nsequence (such as a string, tuple or list) or other iterable object:\n\n for_stmt ::= "for" target_list "in" expression_list ":" suite\n ["else" ":" suite]\n\nThe expression list is evaluated once; it should yield an iterable\nobject. An iterator is created for the result of the\n``expression_list``. The suite is then executed once for each item\nprovided by the iterator, in the order of ascending indices. Each\nitem in turn is assigned to the target list using the standard rules\nfor assignments, and then the suite is executed. When the items are\nexhausted (which is immediately when the sequence is empty), the suite\nin the ``else`` clause, if present, is executed, and the loop\nterminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ncontinues with the next item, or with the ``else`` clause if there was\nno next item.\n\nThe suite may assign to the variable(s) in the target list; this does\nnot affect the next item assigned to it.\n\nThe target list is not deleted when the loop is finished, but if the\nsequence is empty, it will not have been assigned to at all by the\nloop. Hint: the built-in function ``range()`` returns a sequence of\nintegers suitable to emulate the effect of Pascal\'s ``for i := a to b\ndo``; e.g., ``range(3)`` returns the list ``[0, 1, 2]``.\n\nNote: There is a subtlety when the sequence is being modified by the loop\n (this can only occur for mutable sequences, i.e. lists). An internal\n counter is used to keep track of which item is used next, and this\n is incremented on each iteration. When this counter has reached the\n length of the sequence the loop terminates. This means that if the\n suite deletes the current (or a previous) item from the sequence,\n the next item will be skipped (since it gets the index of the\n current item which has already been treated). Likewise, if the\n suite inserts an item in the sequence before the current item, the\n current item will be treated again the next time through the loop.\n This can lead to nasty bugs that can be avoided by making a\n temporary copy using a slice of the whole sequence, e.g.,\n\n for x in a[:]:\n if x < 0: a.remove(x)\n', - 'formatstrings': '\nFormat String Syntax\n********************\n\nThe ``str.format()`` method and the ``Formatter`` class share the same\nsyntax for format strings (although in the case of ``Formatter``,\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n``{}``. Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output. If you need to include\na brace character in the literal text, it can be escaped by doubling:\n``{{`` and ``}}``.\n\nThe grammar for a replacement field is as follows:\n\n replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n field_name ::= arg_name ("." attribute_name | "[" element_index "]")*\n arg_name ::= [identifier | integer]\n attribute_name ::= identifier\n element_index ::= integer | index_string\n index_string ::= +\n conversion ::= "r" | "s"\n format_spec ::= \n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a *conversion* field, which is\npreceded by an exclamation point ``\'!\'``, and a *format_spec*, which\nis preceded by a colon ``\':\'``. These specify a non-default format\nfor the replacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either a\nnumber or a keyword. If it\'s a number, it refers to a positional\nargument, and if it\'s a keyword, it refers to a named keyword\nargument. If the numerical arg_names in a format string are 0, 1, 2,\n... in sequence, they can all be omitted (not just some) and the\nnumbers 0, 1, 2, ... will be automatically inserted in that order.\nBecause *arg_name* is not quote-delimited, it is not possible to\nspecify arbitrary dictionary keys (e.g., the strings ``\'10\'`` or\n``\':-]\'``) within a format string. The *arg_name* can be followed by\nany number of index or attribute expressions. An expression of the\nform ``\'.name\'`` selects the named attribute using ``getattr()``,\nwhile an expression of the form ``\'[index]\'`` does an index lookup\nusing ``__getitem__()``.\n\nChanged in version 2.7: The positional argument specifiers can be\nomitted, so ``\'{} {}\'`` is equivalent to ``\'{0} {1}\'``.\n\nSome simple format string examples:\n\n "First, thou shalt count to {0}" # References first positional argument\n "Bring me a {}" # Implicitly references the first positional argument\n "From {} to {}" # Same as "From {0} to {1}"\n "My quest is {name}" # References keyword argument \'name\'\n "Weight in tons {0.weight}" # \'weight\' attribute of first positional arg\n "Units destroyed: {players[0]}" # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the\n``__format__()`` method of the value itself. However, in some cases\nit is desirable to force a type to be formatted as a string,\noverriding its own definition of formatting. By converting the value\nto a string before calling ``__format__()``, the normal formatting\nlogic is bypassed.\n\nTwo conversion flags are currently supported: ``\'!s\'`` which calls\n``str()`` on the value, and ``\'!r\'`` which calls ``repr()``.\n\nSome examples:\n\n "Harold\'s a clever {0!s}" # Calls str() on the argument first\n "Bring out the holy {name!r}" # Calls repr() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on. Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed. The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*). They can also be passed directly to the\nbuilt-in ``format()`` function. Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string (``""``) produces\nthe same result as if you had called ``str()`` on the value. A non-\nempty format string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n fill ::= \n align ::= "<" | ">" | "=" | "^"\n sign ::= "+" | "-" | " "\n width ::= integer\n precision ::= integer\n type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nThe *fill* character can be any character other than \'{\' or \'}\'. The\npresence of a fill character is signaled by the character following\nit, which must be one of the alignment options. If the second\ncharacter of *format_spec* is not a valid alignment option, then it is\nassumed that both the fill character and the alignment option are\nabsent.\n\nThe meaning of the various alignment options is as follows:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'<\'`` | Forces the field to be left-aligned within the available |\n | | space (this is the default for most objects). |\n +-----------+------------------------------------------------------------+\n | ``\'>\'`` | Forces the field to be right-aligned within the available |\n | | space (this is the default for numbers). |\n +-----------+------------------------------------------------------------+\n | ``\'=\'`` | Forces the padding to be placed after the sign (if any) |\n | | but before the digits. This is used for printing fields |\n | | in the form \'+000000120\'. This alignment option is only |\n | | valid for numeric types. |\n +-----------+------------------------------------------------------------+\n | ``\'^\'`` | Forces the field to be centered within the available |\n | | space. |\n +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'+\'`` | indicates that a sign should be used for both positive as |\n | | well as negative numbers. |\n +-----------+------------------------------------------------------------+\n | ``\'-\'`` | indicates that a sign should be used only for negative |\n | | numbers (this is the default behavior). |\n +-----------+------------------------------------------------------------+\n | space | indicates that a leading space should be used on positive |\n | | numbers, and a minus sign on negative numbers. |\n +-----------+------------------------------------------------------------+\n\nThe ``\'#\'`` option is only valid for integers, and only for binary,\noctal, or hexadecimal output. If present, it specifies that the\noutput will be prefixed by ``\'0b\'``, ``\'0o\'``, or ``\'0x\'``,\nrespectively.\n\nThe ``\',\'`` option signals the use of a comma for a thousands\nseparator. For a locale aware separator, use the ``\'n\'`` integer\npresentation type instead.\n\nChanged in version 2.7: Added the ``\',\'`` option (see also **PEP\n378**).\n\n*width* is a decimal integer defining the minimum field width. If not\nspecified, then the field width will be determined by the content.\n\nPreceding the *width* field by a zero (``\'0\'``) character enables\nsign-aware zero-padding for numeric types. This is equivalent to a\n*fill* character of ``\'0\'`` with an *alignment* type of ``\'=\'``.\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with ``\'f\'`` and ``\'F\'``, or before and after the decimal\npoint for a floating point value formatted with ``\'g\'`` or ``\'G\'``.\nFor non-number types the field indicates the maximum field size - in\nother words, how many characters will be used from the field content.\nThe *precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'s\'`` | String format. This is the default type for strings and |\n | | may be omitted. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'s\'``. |\n +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'b\'`` | Binary format. Outputs the number in base 2. |\n +-----------+------------------------------------------------------------+\n | ``\'c\'`` | Character. Converts the integer to the corresponding |\n | | unicode character before printing. |\n +-----------+------------------------------------------------------------+\n | ``\'d\'`` | Decimal Integer. Outputs the number in base 10. |\n +-----------+------------------------------------------------------------+\n | ``\'o\'`` | Octal format. Outputs the number in base 8. |\n +-----------+------------------------------------------------------------+\n | ``\'x\'`` | Hex format. Outputs the number in base 16, using lower- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'X\'`` | Hex format. Outputs the number in base 16, using upper- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'d\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'d\'``. |\n +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except\n``\'n\'`` and None). When doing so, ``float()`` is used to convert the\ninteger to a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'e\'`` | Exponent notation. Prints the number in scientific |\n | | notation using the letter \'e\' to indicate the exponent. |\n +-----------+------------------------------------------------------------+\n | ``\'E\'`` | Exponent notation. Same as ``\'e\'`` except it uses an upper |\n | | case \'E\' as the separator character. |\n +-----------+------------------------------------------------------------+\n | ``\'f\'`` | Fixed point. Displays the number as a fixed-point number. |\n +-----------+------------------------------------------------------------+\n | ``\'F\'`` | Fixed point. Same as ``\'f\'``. |\n +-----------+------------------------------------------------------------+\n | ``\'g\'`` | General format. For a given precision ``p >= 1``, this |\n | | rounds the number to ``p`` significant digits and then |\n | | formats the result in either fixed-point format or in |\n | | scientific notation, depending on its magnitude. The |\n | | precise rules are as follows: suppose that the result |\n | | formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1`` would have exponent ``exp``. Then if ``-4 <= exp |\n | | < p``, the number is formatted with presentation type |\n | | ``\'f\'`` and precision ``p-1-exp``. Otherwise, the number |\n | | is formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1``. In both cases insignificant trailing zeros are |\n | | removed from the significand, and the decimal point is |\n | | also removed if there are no remaining digits following |\n | | it. Positive and negative infinity, positive and negative |\n | | zero, and nans, are formatted as ``inf``, ``-inf``, ``0``, |\n | | ``-0`` and ``nan`` respectively, regardless of the |\n | | precision. A precision of ``0`` is treated as equivalent |\n | | to a precision of ``1``. |\n +-----------+------------------------------------------------------------+\n | ``\'G\'`` | General format. Same as ``\'g\'`` except switches to ``\'E\'`` |\n | | if the number gets too large. The representations of |\n | | infinity and NaN are uppercased, too. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'g\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | ``\'%\'`` | Percentage. Multiplies the number by 100 and displays in |\n | | fixed (``\'f\'``) format, followed by a percent sign. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'g\'``. |\n +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old ``%``-formatting.\n\nIn most of the cases the syntax is similar to the old\n``%``-formatting, with the addition of the ``{}`` and with ``:`` used\ninstead of ``%``. For example, ``\'%03.2f\'`` can be translated to\n``\'{:03.2f}\'``.\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n \'a, b, c\'\n >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\') # 2.7+ only\n \'a, b, c\'\n >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n \'c, b, a\'\n >>> \'{2}, {1}, {0}\'.format(*\'abc\') # unpacking argument sequence\n \'c, b, a\'\n >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\') # arguments\' indices can be repeated\n \'abracadabra\'\n\nAccessing arguments by name:\n\n >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n \'Coordinates: 37.24N, -115.81W\'\n >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n >>> c = 3-5j\n >>> (\'The complex number {0} is formed from the real part {0.real} \'\n ... \'and the imaginary part {0.imag}.\').format(c)\n \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n >>> class Point(object):\n ... def __init__(self, x, y):\n ... self.x, self.y = x, y\n ... def __str__(self):\n ... return \'Point({self.x}, {self.y})\'.format(self=self)\n ...\n >>> str(Point(4, 2))\n \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n >>> coord = (3, 5)\n >>> \'X: {0[0]}; Y: {0[1]}\'.format(coord)\n \'X: 3; Y: 5\'\n\nReplacing ``%s`` and ``%r``:\n\n >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n >>> \'{:<30}\'.format(\'left aligned\')\n \'left aligned \'\n >>> \'{:>30}\'.format(\'right aligned\')\n \' right aligned\'\n >>> \'{:^30}\'.format(\'centered\')\n \' centered \'\n >>> \'{:*^30}\'.format(\'centered\') # use \'*\' as a fill char\n \'***********centered***********\'\n\nReplacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:\n\n >>> \'{:+f}; {:+f}\'.format(3.14, -3.14) # show it always\n \'+3.140000; -3.140000\'\n >>> \'{: f}; {: f}\'.format(3.14, -3.14) # show a space for positive numbers\n \' 3.140000; -3.140000\'\n >>> \'{:-f}; {:-f}\'.format(3.14, -3.14) # show only the minus -- same as \'{:f}; {:f}\'\n \'3.140000; -3.140000\'\n\nReplacing ``%x`` and ``%o`` and converting the value to different\nbases:\n\n >>> # format also supports binary numbers\n >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42)\n \'int: 42; hex: 2a; oct: 52; bin: 101010\'\n >>> # with 0x, 0o, or 0b as prefix:\n >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42)\n \'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n >>> \'{:,}\'.format(1234567890)\n \'1,234,567,890\'\n\nExpressing a percentage:\n\n >>> points = 19.5\n >>> total = 22\n >>> \'Correct answers: {:.2%}\'.format(points/total)\n \'Correct answers: 88.64%\'\n\nUsing type-specific formatting:\n\n >>> import datetime\n >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n ... \'{0:{fill}{align}16}\'.format(text, fill=align, align=align)\n ...\n \'left<<<<<<<<<<<<\'\n \'^^^^^center^^^^^\'\n \'>>>>>>>>>>>right\'\n >>>\n >>> octets = [192, 168, 0, 1]\n >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n \'C0A80001\'\n >>> int(_, 16)\n 3232235521\n >>>\n >>> width = 5\n >>> for num in range(5,12):\n ... for base in \'dXob\':\n ... print \'{0:{width}{base}}\'.format(num, base=base, width=width),\n ... print\n ...\n 5 5 5 101\n 6 6 6 110\n 7 7 7 111\n 8 8 10 1000\n 9 9 11 1001\n 10 A 12 1010\n 11 B 13 1011\n', + 'formatstrings': '\nFormat String Syntax\n********************\n\nThe ``str.format()`` method and the ``Formatter`` class share the same\nsyntax for format strings (although in the case of ``Formatter``,\nsubclasses can define their own format string syntax).\n\nFormat strings contain "replacement fields" surrounded by curly braces\n``{}``. Anything that is not contained in braces is considered literal\ntext, which is copied unchanged to the output. If you need to include\na brace character in the literal text, it can be escaped by doubling:\n``{{`` and ``}}``.\n\nThe grammar for a replacement field is as follows:\n\n replacement_field ::= "{" [field_name] ["!" conversion] [":" format_spec] "}"\n field_name ::= arg_name ("." attribute_name | "[" element_index "]")*\n arg_name ::= [identifier | integer]\n attribute_name ::= identifier\n element_index ::= integer | index_string\n index_string ::= +\n conversion ::= "r" | "s"\n format_spec ::= \n\nIn less formal terms, the replacement field can start with a\n*field_name* that specifies the object whose value is to be formatted\nand inserted into the output instead of the replacement field. The\n*field_name* is optionally followed by a *conversion* field, which is\npreceded by an exclamation point ``\'!\'``, and a *format_spec*, which\nis preceded by a colon ``\':\'``. These specify a non-default format\nfor the replacement value.\n\nSee also the *Format Specification Mini-Language* section.\n\nThe *field_name* itself begins with an *arg_name* that is either a\nnumber or a keyword. If it\'s a number, it refers to a positional\nargument, and if it\'s a keyword, it refers to a named keyword\nargument. If the numerical arg_names in a format string are 0, 1, 2,\n... in sequence, they can all be omitted (not just some) and the\nnumbers 0, 1, 2, ... will be automatically inserted in that order.\nBecause *arg_name* is not quote-delimited, it is not possible to\nspecify arbitrary dictionary keys (e.g., the strings ``\'10\'`` or\n``\':-]\'``) within a format string. The *arg_name* can be followed by\nany number of index or attribute expressions. An expression of the\nform ``\'.name\'`` selects the named attribute using ``getattr()``,\nwhile an expression of the form ``\'[index]\'`` does an index lookup\nusing ``__getitem__()``.\n\nChanged in version 2.7: The positional argument specifiers can be\nomitted, so ``\'{} {}\'`` is equivalent to ``\'{0} {1}\'``.\n\nSome simple format string examples:\n\n "First, thou shalt count to {0}" # References first positional argument\n "Bring me a {}" # Implicitly references the first positional argument\n "From {} to {}" # Same as "From {0} to {1}"\n "My quest is {name}" # References keyword argument \'name\'\n "Weight in tons {0.weight}" # \'weight\' attribute of first positional arg\n "Units destroyed: {players[0]}" # First element of keyword argument \'players\'.\n\nThe *conversion* field causes a type coercion before formatting.\nNormally, the job of formatting a value is done by the\n``__format__()`` method of the value itself. However, in some cases\nit is desirable to force a type to be formatted as a string,\noverriding its own definition of formatting. By converting the value\nto a string before calling ``__format__()``, the normal formatting\nlogic is bypassed.\n\nTwo conversion flags are currently supported: ``\'!s\'`` which calls\n``str()`` on the value, and ``\'!r\'`` which calls ``repr()``.\n\nSome examples:\n\n "Harold\'s a clever {0!s}" # Calls str() on the argument first\n "Bring out the holy {name!r}" # Calls repr() on the argument first\n\nThe *format_spec* field contains a specification of how the value\nshould be presented, including such details as field width, alignment,\npadding, decimal precision and so on. Each value type can define its\nown "formatting mini-language" or interpretation of the *format_spec*.\n\nMost built-in types support a common formatting mini-language, which\nis described in the next section.\n\nA *format_spec* field can also include nested replacement fields\nwithin it. These nested replacement fields can contain only a field\nname; conversion flags and format specifications are not allowed. The\nreplacement fields within the format_spec are substituted before the\n*format_spec* string is interpreted. This allows the formatting of a\nvalue to be dynamically specified.\n\nSee the *Format examples* section for some examples.\n\n\nFormat Specification Mini-Language\n==================================\n\n"Format specifications" are used within replacement fields contained\nwithin a format string to define how individual values are presented\n(see *Format String Syntax*). They can also be passed directly to the\nbuilt-in ``format()`` function. Each formattable type may define how\nthe format specification is to be interpreted.\n\nMost built-in types implement the following options for format\nspecifications, although some of the formatting options are only\nsupported by the numeric types.\n\nA general convention is that an empty format string (``""``) produces\nthe same result as if you had called ``str()`` on the value. A non-\nempty format string typically modifies the result.\n\nThe general form of a *standard format specifier* is:\n\n format_spec ::= [[fill]align][sign][#][0][width][,][.precision][type]\n fill ::= \n align ::= "<" | ">" | "=" | "^"\n sign ::= "+" | "-" | " "\n width ::= integer\n precision ::= integer\n type ::= "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%"\n\nThe *fill* character can be any character other than \'{\' or \'}\'. The\npresence of a fill character is signaled by the character following\nit, which must be one of the alignment options. If the second\ncharacter of *format_spec* is not a valid alignment option, then it is\nassumed that both the fill character and the alignment option are\nabsent.\n\nThe meaning of the various alignment options is as follows:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'<\'`` | Forces the field to be left-aligned within the available |\n | | space (this is the default for most objects). |\n +-----------+------------------------------------------------------------+\n | ``\'>\'`` | Forces the field to be right-aligned within the available |\n | | space (this is the default for numbers). |\n +-----------+------------------------------------------------------------+\n | ``\'=\'`` | Forces the padding to be placed after the sign (if any) |\n | | but before the digits. This is used for printing fields |\n | | in the form \'+000000120\'. This alignment option is only |\n | | valid for numeric types. |\n +-----------+------------------------------------------------------------+\n | ``\'^\'`` | Forces the field to be centered within the available |\n | | space. |\n +-----------+------------------------------------------------------------+\n\nNote that unless a minimum field width is defined, the field width\nwill always be the same size as the data to fill it, so that the\nalignment option has no meaning in this case.\n\nThe *sign* option is only valid for number types, and can be one of\nthe following:\n\n +-----------+------------------------------------------------------------+\n | Option | Meaning |\n +===========+============================================================+\n | ``\'+\'`` | indicates that a sign should be used for both positive as |\n | | well as negative numbers. |\n +-----------+------------------------------------------------------------+\n | ``\'-\'`` | indicates that a sign should be used only for negative |\n | | numbers (this is the default behavior). |\n +-----------+------------------------------------------------------------+\n | space | indicates that a leading space should be used on positive |\n | | numbers, and a minus sign on negative numbers. |\n +-----------+------------------------------------------------------------+\n\nThe ``\'#\'`` option is only valid for integers, and only for binary,\noctal, or hexadecimal output. If present, it specifies that the\noutput will be prefixed by ``\'0b\'``, ``\'0o\'``, or ``\'0x\'``,\nrespectively.\n\nThe ``\',\'`` option signals the use of a comma for a thousands\nseparator. For a locale aware separator, use the ``\'n\'`` integer\npresentation type instead.\n\nChanged in version 2.7: Added the ``\',\'`` option (see also **PEP\n378**).\n\n*width* is a decimal integer defining the minimum field width. If not\nspecified, then the field width will be determined by the content.\n\nPreceding the *width* field by a zero (``\'0\'``) character enables\nsign-aware zero-padding for numeric types. This is equivalent to a\n*fill* character of ``\'0\'`` with an *alignment* type of ``\'=\'``.\n\nThe *precision* is a decimal number indicating how many digits should\nbe displayed after the decimal point for a floating point value\nformatted with ``\'f\'`` and ``\'F\'``, or before and after the decimal\npoint for a floating point value formatted with ``\'g\'`` or ``\'G\'``.\nFor non-number types the field indicates the maximum field size - in\nother words, how many characters will be used from the field content.\nThe *precision* is not allowed for integer values.\n\nFinally, the *type* determines how the data should be presented.\n\nThe available string presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'s\'`` | String format. This is the default type for strings and |\n | | may be omitted. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'s\'``. |\n +-----------+------------------------------------------------------------+\n\nThe available integer presentation types are:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'b\'`` | Binary format. Outputs the number in base 2. |\n +-----------+------------------------------------------------------------+\n | ``\'c\'`` | Character. Converts the integer to the corresponding |\n | | unicode character before printing. |\n +-----------+------------------------------------------------------------+\n | ``\'d\'`` | Decimal Integer. Outputs the number in base 10. |\n +-----------+------------------------------------------------------------+\n | ``\'o\'`` | Octal format. Outputs the number in base 8. |\n +-----------+------------------------------------------------------------+\n | ``\'x\'`` | Hex format. Outputs the number in base 16, using lower- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'X\'`` | Hex format. Outputs the number in base 16, using upper- |\n | | case letters for the digits above 9. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'d\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'d\'``. |\n +-----------+------------------------------------------------------------+\n\nIn addition to the above presentation types, integers can be formatted\nwith the floating point presentation types listed below (except\n``\'n\'`` and None). When doing so, ``float()`` is used to convert the\ninteger to a floating point number before formatting.\n\nThe available presentation types for floating point and decimal values\nare:\n\n +-----------+------------------------------------------------------------+\n | Type | Meaning |\n +===========+============================================================+\n | ``\'e\'`` | Exponent notation. Prints the number in scientific |\n | | notation using the letter \'e\' to indicate the exponent. |\n | | The default precision is ``6``. |\n +-----------+------------------------------------------------------------+\n | ``\'E\'`` | Exponent notation. Same as ``\'e\'`` except it uses an upper |\n | | case \'E\' as the separator character. |\n +-----------+------------------------------------------------------------+\n | ``\'f\'`` | Fixed point. Displays the number as a fixed-point number. |\n | | The default precision is ``6``. |\n +-----------+------------------------------------------------------------+\n | ``\'F\'`` | Fixed point. Same as ``\'f\'``. |\n +-----------+------------------------------------------------------------+\n | ``\'g\'`` | General format. For a given precision ``p >= 1``, this |\n | | rounds the number to ``p`` significant digits and then |\n | | formats the result in either fixed-point format or in |\n | | scientific notation, depending on its magnitude. The |\n | | precise rules are as follows: suppose that the result |\n | | formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1`` would have exponent ``exp``. Then if ``-4 <= exp |\n | | < p``, the number is formatted with presentation type |\n | | ``\'f\'`` and precision ``p-1-exp``. Otherwise, the number |\n | | is formatted with presentation type ``\'e\'`` and precision |\n | | ``p-1``. In both cases insignificant trailing zeros are |\n | | removed from the significand, and the decimal point is |\n | | also removed if there are no remaining digits following |\n | | it. Positive and negative infinity, positive and negative |\n | | zero, and nans, are formatted as ``inf``, ``-inf``, ``0``, |\n | | ``-0`` and ``nan`` respectively, regardless of the |\n | | precision. A precision of ``0`` is treated as equivalent |\n | | to a precision of ``1``. The default precision is ``6``. |\n +-----------+------------------------------------------------------------+\n | ``\'G\'`` | General format. Same as ``\'g\'`` except switches to ``\'E\'`` |\n | | if the number gets too large. The representations of |\n | | infinity and NaN are uppercased, too. |\n +-----------+------------------------------------------------------------+\n | ``\'n\'`` | Number. This is the same as ``\'g\'``, except that it uses |\n | | the current locale setting to insert the appropriate |\n | | number separator characters. |\n +-----------+------------------------------------------------------------+\n | ``\'%\'`` | Percentage. Multiplies the number by 100 and displays in |\n | | fixed (``\'f\'``) format, followed by a percent sign. |\n +-----------+------------------------------------------------------------+\n | None | The same as ``\'g\'``. |\n +-----------+------------------------------------------------------------+\n\n\nFormat examples\n===============\n\nThis section contains examples of the new format syntax and comparison\nwith the old ``%``-formatting.\n\nIn most of the cases the syntax is similar to the old\n``%``-formatting, with the addition of the ``{}`` and with ``:`` used\ninstead of ``%``. For example, ``\'%03.2f\'`` can be translated to\n``\'{:03.2f}\'``.\n\nThe new format syntax also supports new and different options, shown\nin the follow examples.\n\nAccessing arguments by position:\n\n >>> \'{0}, {1}, {2}\'.format(\'a\', \'b\', \'c\')\n \'a, b, c\'\n >>> \'{}, {}, {}\'.format(\'a\', \'b\', \'c\') # 2.7+ only\n \'a, b, c\'\n >>> \'{2}, {1}, {0}\'.format(\'a\', \'b\', \'c\')\n \'c, b, a\'\n >>> \'{2}, {1}, {0}\'.format(*\'abc\') # unpacking argument sequence\n \'c, b, a\'\n >>> \'{0}{1}{0}\'.format(\'abra\', \'cad\') # arguments\' indices can be repeated\n \'abracadabra\'\n\nAccessing arguments by name:\n\n >>> \'Coordinates: {latitude}, {longitude}\'.format(latitude=\'37.24N\', longitude=\'-115.81W\')\n \'Coordinates: 37.24N, -115.81W\'\n >>> coord = {\'latitude\': \'37.24N\', \'longitude\': \'-115.81W\'}\n >>> \'Coordinates: {latitude}, {longitude}\'.format(**coord)\n \'Coordinates: 37.24N, -115.81W\'\n\nAccessing arguments\' attributes:\n\n >>> c = 3-5j\n >>> (\'The complex number {0} is formed from the real part {0.real} \'\n ... \'and the imaginary part {0.imag}.\').format(c)\n \'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.\'\n >>> class Point(object):\n ... def __init__(self, x, y):\n ... self.x, self.y = x, y\n ... def __str__(self):\n ... return \'Point({self.x}, {self.y})\'.format(self=self)\n ...\n >>> str(Point(4, 2))\n \'Point(4, 2)\'\n\nAccessing arguments\' items:\n\n >>> coord = (3, 5)\n >>> \'X: {0[0]}; Y: {0[1]}\'.format(coord)\n \'X: 3; Y: 5\'\n\nReplacing ``%s`` and ``%r``:\n\n >>> "repr() shows quotes: {!r}; str() doesn\'t: {!s}".format(\'test1\', \'test2\')\n "repr() shows quotes: \'test1\'; str() doesn\'t: test2"\n\nAligning the text and specifying a width:\n\n >>> \'{:<30}\'.format(\'left aligned\')\n \'left aligned \'\n >>> \'{:>30}\'.format(\'right aligned\')\n \' right aligned\'\n >>> \'{:^30}\'.format(\'centered\')\n \' centered \'\n >>> \'{:*^30}\'.format(\'centered\') # use \'*\' as a fill char\n \'***********centered***********\'\n\nReplacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:\n\n >>> \'{:+f}; {:+f}\'.format(3.14, -3.14) # show it always\n \'+3.140000; -3.140000\'\n >>> \'{: f}; {: f}\'.format(3.14, -3.14) # show a space for positive numbers\n \' 3.140000; -3.140000\'\n >>> \'{:-f}; {:-f}\'.format(3.14, -3.14) # show only the minus -- same as \'{:f}; {:f}\'\n \'3.140000; -3.140000\'\n\nReplacing ``%x`` and ``%o`` and converting the value to different\nbases:\n\n >>> # format also supports binary numbers\n >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42)\n \'int: 42; hex: 2a; oct: 52; bin: 101010\'\n >>> # with 0x, 0o, or 0b as prefix:\n >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42)\n \'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010\'\n\nUsing the comma as a thousands separator:\n\n >>> \'{:,}\'.format(1234567890)\n \'1,234,567,890\'\n\nExpressing a percentage:\n\n >>> points = 19.5\n >>> total = 22\n >>> \'Correct answers: {:.2%}\'.format(points/total)\n \'Correct answers: 88.64%\'\n\nUsing type-specific formatting:\n\n >>> import datetime\n >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58)\n >>> \'{:%Y-%m-%d %H:%M:%S}\'.format(d)\n \'2010-07-04 12:15:58\'\n\nNesting arguments and more complex examples:\n\n >>> for align, text in zip(\'<^>\', [\'left\', \'center\', \'right\']):\n ... \'{0:{fill}{align}16}\'.format(text, fill=align, align=align)\n ...\n \'left<<<<<<<<<<<<\'\n \'^^^^^center^^^^^\'\n \'>>>>>>>>>>>right\'\n >>>\n >>> octets = [192, 168, 0, 1]\n >>> \'{:02X}{:02X}{:02X}{:02X}\'.format(*octets)\n \'C0A80001\'\n >>> int(_, 16)\n 3232235521\n >>>\n >>> width = 5\n >>> for num in range(5,12):\n ... for base in \'dXob\':\n ... print \'{0:{width}{base}}\'.format(num, base=base, width=width),\n ... print\n ...\n 5 5 5 101\n 6 6 6 110\n 7 7 7 111\n 8 8 10 1000\n 9 9 11 1001\n 10 A 12 1010\n 11 B 13 1011\n', 'function': '\nFunction definitions\n********************\n\nA function definition defines a user-defined function object (see\nsection *The standard type hierarchy*):\n\n decorated ::= decorators (classdef | funcdef)\n decorators ::= decorator+\n decorator ::= "@" dotted_name ["(" [argument_list [","]] ")"] NEWLINE\n funcdef ::= "def" funcname "(" [parameter_list] ")" ":" suite\n dotted_name ::= identifier ("." identifier)*\n parameter_list ::= (defparameter ",")*\n ( "*" identifier ["," "**" identifier]\n | "**" identifier\n | defparameter [","] )\n defparameter ::= parameter ["=" expression]\n sublist ::= parameter ("," parameter)* [","]\n parameter ::= identifier | "(" sublist ")"\n funcname ::= identifier\n\nA function definition is an executable statement. Its execution binds\nthe function name in the current local namespace to a function object\n(a wrapper around the executable code for the function). This\nfunction object contains a reference to the current global namespace\nas the global namespace to be used when the function is called.\n\nThe function definition does not execute the function body; this gets\nexecuted only when the function is called. [3]\n\nA function definition may be wrapped by one or more *decorator*\nexpressions. Decorator expressions are evaluated when the function is\ndefined, in the scope that contains the function definition. The\nresult must be a callable, which is invoked with the function object\nas the only argument. The returned value is bound to the function name\ninstead of the function object. Multiple decorators are applied in\nnested fashion. For example, the following code:\n\n @f1(arg)\n @f2\n def func(): pass\n\nis equivalent to:\n\n def func(): pass\n func = f1(arg)(f2(func))\n\nWhen one or more top-level *parameters* have the form *parameter*\n``=`` *expression*, the function is said to have "default parameter\nvalues." For a parameter with a default value, the corresponding\n*argument* may be omitted from a call, in which case the parameter\'s\ndefault value is substituted. If a parameter has a default value, all\nfollowing parameters must also have a default value --- this is a\nsyntactic restriction that is not expressed by the grammar.\n\n**Default parameter values are evaluated when the function definition\nis executed.** This means that the expression is evaluated once, when\nthe function is defined, and that the same "pre-computed" value is\nused for each call. This is especially important to understand when a\ndefault parameter is a mutable object, such as a list or a dictionary:\nif the function modifies the object (e.g. by appending an item to a\nlist), the default value is in effect modified. This is generally not\nwhat was intended. A way around this is to use ``None`` as the\ndefault, and explicitly test for it in the body of the function, e.g.:\n\n def whats_on_the_telly(penguin=None):\n if penguin is None:\n penguin = []\n penguin.append("property of the zoo")\n return penguin\n\nFunction call semantics are described in more detail in section\n*Calls*. A function call always assigns values to all parameters\nmentioned in the parameter list, either from position arguments, from\nkeyword arguments, or from default values. If the form\n"``*identifier``" is present, it is initialized to a tuple receiving\nany excess positional parameters, defaulting to the empty tuple. If\nthe form "``**identifier``" is present, it is initialized to a new\ndictionary receiving any excess keyword arguments, defaulting to a new\nempty dictionary.\n\nIt is also possible to create anonymous functions (functions not bound\nto a name), for immediate use in expressions. This uses lambda forms,\ndescribed in section *Lambdas*. Note that the lambda form is merely a\nshorthand for a simplified function definition; a function defined in\na "``def``" statement can be passed around or assigned to another name\njust like a function defined by a lambda form. The "``def``" form is\nactually more powerful since it allows the execution of multiple\nstatements.\n\n**Programmer\'s note:** Functions are first-class objects. A "``def``"\nform executed inside a function definition defines a local function\nthat can be returned or passed around. Free variables used in the\nnested function can access the local variables of the function\ncontaining the def. See section *Naming and binding* for details.\n', 'global': '\nThe ``global`` statement\n************************\n\n global_stmt ::= "global" identifier ("," identifier)*\n\nThe ``global`` statement is a declaration which holds for the entire\ncurrent code block. It means that the listed identifiers are to be\ninterpreted as globals. It would be impossible to assign to a global\nvariable without ``global``, although free variables may refer to\nglobals without being declared global.\n\nNames listed in a ``global`` statement must not be used in the same\ncode block textually preceding that ``global`` statement.\n\nNames listed in a ``global`` statement must not be defined as formal\nparameters or in a ``for`` loop control target, ``class`` definition,\nfunction definition, or ``import`` statement.\n\n**CPython implementation detail:** The current implementation does not\nenforce the latter two restrictions, but programs should not abuse\nthis freedom, as future implementations may enforce them or silently\nchange the meaning of the program.\n\n**Programmer\'s note:** the ``global`` is a directive to the parser.\nIt applies only to code parsed at the same time as the ``global``\nstatement. In particular, a ``global`` statement contained in an\n``exec`` statement does not affect the code block *containing* the\n``exec`` statement, and code contained in an ``exec`` statement is\nunaffected by ``global`` statements in the code containing the\n``exec`` statement. The same applies to the ``eval()``,\n``execfile()`` and ``compile()`` functions.\n', 'id-classes': '\nReserved classes of identifiers\n*******************************\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library). Current\n system names are discussed in the *Special method names* section\n and elsewhere. More will likely be defined in future versions of\n Python. *Any* use of ``__*__`` names, in any context, that does\n not follow explicitly documented use, is subject to breakage\n without warning.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', 'identifiers': '\nIdentifiers and keywords\n************************\n\nIdentifiers (also referred to as *names*) are described by the\nfollowing lexical definitions:\n\n identifier ::= (letter|"_") (letter | digit | "_")*\n letter ::= lowercase | uppercase\n lowercase ::= "a"..."z"\n uppercase ::= "A"..."Z"\n digit ::= "0"..."9"\n\nIdentifiers are unlimited in length. Case is significant.\n\n\nKeywords\n========\n\nThe following identifiers are used as reserved words, or *keywords* of\nthe language, and cannot be used as ordinary identifiers. They must\nbe spelled exactly as written here:\n\n and del from not while\n as elif global or with\n assert else if pass yield\n break except import print\n class exec in raise\n continue finally is return\n def for lambda try\n\nChanged in version 2.4: ``None`` became a constant and is now\nrecognized by the compiler as a name for the built-in object ``None``.\nAlthough it is not a keyword, you cannot assign a different object to\nit.\n\nChanged in version 2.5: Using ``as`` and ``with`` as identifiers\ntriggers a warning. To use them as keywords, enable the\n``with_statement`` future feature .\n\nChanged in version 2.6: ``as`` and ``with`` are full keywords.\n\n\nReserved classes of identifiers\n===============================\n\nCertain classes of identifiers (besides keywords) have special\nmeanings. These classes are identified by the patterns of leading and\ntrailing underscore characters:\n\n``_*``\n Not imported by ``from module import *``. The special identifier\n ``_`` is used in the interactive interpreter to store the result of\n the last evaluation; it is stored in the ``__builtin__`` module.\n When not in interactive mode, ``_`` has no special meaning and is\n not defined. See section *The import statement*.\n\n Note: The name ``_`` is often used in conjunction with\n internationalization; refer to the documentation for the\n ``gettext`` module for more information on this convention.\n\n``__*__``\n System-defined names. These names are defined by the interpreter\n and its implementation (including the standard library). Current\n system names are discussed in the *Special method names* section\n and elsewhere. More will likely be defined in future versions of\n Python. *Any* use of ``__*__`` names, in any context, that does\n not follow explicitly documented use, is subject to breakage\n without warning.\n\n``__*``\n Class-private names. Names in this category, when used within the\n context of a class definition, are re-written to use a mangled form\n to help avoid name clashes between "private" attributes of base and\n derived classes. See section *Identifiers (Names)*.\n', 'if': '\nThe ``if`` statement\n********************\n\nThe ``if`` statement is used for conditional execution:\n\n if_stmt ::= "if" expression ":" suite\n ( "elif" expression ":" suite )*\n ["else" ":" suite]\n\nIt selects exactly one of the suites by evaluating the expressions one\nby one until one is found to be true (see section *Boolean operations*\nfor the definition of true and false); then that suite is executed\n(and no other part of the ``if`` statement is executed or evaluated).\nIf all expressions are false, the suite of the ``else`` clause, if\npresent, is executed.\n', 'imaginary': '\nImaginary literals\n******************\n\nImaginary literals are described by the following lexical definitions:\n\n imagnumber ::= (floatnumber | intpart) ("j" | "J")\n\nAn imaginary literal yields a complex number with a real part of 0.0.\nComplex numbers are represented as a pair of floating point numbers\nand have the same restrictions on their range. To create a complex\nnumber with a nonzero real part, add a floating point number to it,\ne.g., ``(3+4j)``. Some examples of imaginary literals:\n\n 3.14j 10.j 10j .001j 1e100j 3.14e-10j\n', - 'import': '\nThe ``import`` statement\n************************\n\n import_stmt ::= "import" module ["as" name] ( "," module ["as" name] )*\n | "from" relative_module "import" identifier ["as" name]\n ( "," identifier ["as" name] )*\n | "from" relative_module "import" "(" identifier ["as" name]\n ( "," identifier ["as" name] )* [","] ")"\n | "from" module "import" "*"\n module ::= (identifier ".")* identifier\n relative_module ::= "."* module | "."+\n name ::= identifier\n\nImport statements are executed in two steps: (1) find a module, and\ninitialize it if necessary; (2) define a name or names in the local\nnamespace (of the scope where the ``import`` statement occurs). The\nstatement comes in two forms differing on whether it uses the ``from``\nkeyword. The first form (without ``from``) repeats these steps for\neach identifier in the list. The form with ``from`` performs step (1)\nonce, and then performs step (2) repeatedly.\n\nTo understand how step (1) occurs, one must first understand how\nPython handles hierarchical naming of modules. To help organize\nmodules and provide a hierarchy in naming, Python has a concept of\npackages. A package can contain other packages and modules while\nmodules cannot contain other modules or packages. From a file system\nperspective, packages are directories and modules are files. The\noriginal specification for packages is still available to read,\nalthough minor details have changed since the writing of that\ndocument.\n\nOnce the name of the module is known (unless otherwise specified, the\nterm "module" will refer to both packages and modules), searching for\nthe module or package can begin. The first place checked is\n``sys.modules``, the cache of all modules that have been imported\npreviously. If the module is found there then it is used in step (2)\nof import.\n\nIf the module is not found in the cache, then ``sys.meta_path`` is\nsearched (the specification for ``sys.meta_path`` can be found in\n**PEP 302**). The object is a list of *finder* objects which are\nqueried in order as to whether they know how to load the module by\ncalling their ``find_module()`` method with the name of the module. If\nthe module happens to be contained within a package (as denoted by the\nexistence of a dot in the name), then a second argument to\n``find_module()`` is given as the value of the ``__path__`` attribute\nfrom the parent package (everything up to the last dot in the name of\nthe module being imported). If a finder can find the module it returns\na *loader* (discussed later) or returns ``None``.\n\nIf none of the finders on ``sys.meta_path`` are able to find the\nmodule then some implicitly defined finders are queried.\nImplementations of Python vary in what implicit meta path finders are\ndefined. The one they all do define, though, is one that handles\n``sys.path_hooks``, ``sys.path_importer_cache``, and ``sys.path``.\n\nThe implicit finder searches for the requested module in the "paths"\nspecified in one of two places ("paths" do not have to be file system\npaths). If the module being imported is supposed to be contained\nwithin a package then the second argument passed to ``find_module()``,\n``__path__`` on the parent package, is used as the source of paths. If\nthe module is not contained in a package then ``sys.path`` is used as\nthe source of paths.\n\nOnce the source of paths is chosen it is iterated over to find a\nfinder that can handle that path. The dict at\n``sys.path_importer_cache`` caches finders for paths and is checked\nfor a finder. If the path does not have a finder cached then\n``sys.path_hooks`` is searched by calling each object in the list with\na single argument of the path, returning a finder or raises\n``ImportError``. If a finder is returned then it is cached in\n``sys.path_importer_cache`` and then used for that path entry. If no\nfinder can be found but the path exists then a value of ``None`` is\nstored in ``sys.path_importer_cache`` to signify that an implicit,\nfile-based finder that handles modules stored as individual files\nshould be used for that path. If the path does not exist then a finder\nwhich always returns *None`* is placed in the cache for the path.\n\nIf no finder can find the module then ``ImportError`` is raised.\nOtherwise some finder returned a loader whose ``load_module()`` method\nis called with the name of the module to load (see **PEP 302** for the\noriginal definition of loaders). A loader has several responsibilities\nto perform on a module it loads. First, if the module already exists\nin ``sys.modules`` (a possibility if the loader is called outside of\nthe import machinery) then it is to use that module for initialization\nand not a new module. But if the module does not exist in\n``sys.modules`` then it is to be added to that dict before\ninitialization begins. If an error occurs during loading of the module\nand it was added to ``sys.modules`` it is to be removed from the dict.\nIf an error occurs but the module was already in ``sys.modules`` it is\nleft in the dict.\n\nThe loader must set several attributes on the module. ``__name__`` is\nto be set to the name of the module. ``__file__`` is to be the "path"\nto the file unless the module is built-in (and thus listed in\n``sys.builtin_module_names``) in which case the attribute is not set.\nIf what is being imported is a package then ``__path__`` is to be set\nto a list of paths to be searched when looking for modules and\npackages contained within the package being imported. ``__package__``\nis optional but should be set to the name of package that contains the\nmodule or package (the empty string is used for module not contained\nin a package). ``__loader__`` is also optional but should be set to\nthe loader object that is loading the module.\n\nIf an error occurs during loading then the loader raises\n``ImportError`` if some other exception is not already being\npropagated. Otherwise the loader returns the module that was loaded\nand initialized.\n\nWhen step (1) finishes without raising an exception, step (2) can\nbegin.\n\nThe first form of ``import`` statement binds the module name in the\nlocal namespace to the module object, and then goes on to import the\nnext identifier, if any. If the module name is followed by ``as``,\nthe name following ``as`` is used as the local name for the module.\n\nThe ``from`` form does not bind the module name: it goes through the\nlist of identifiers, looks each one of them up in the module found in\nstep (1), and binds the name in the local namespace to the object thus\nfound. As with the first form of ``import``, an alternate local name\ncan be supplied by specifying "``as`` localname". If a name is not\nfound, ``ImportError`` is raised. If the list of identifiers is\nreplaced by a star (``\'*\'``), all public names defined in the module\nare bound in the local namespace of the ``import`` statement..\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named ``__all__``; if defined, it\nmust be a sequence of strings which are names defined or imported by\nthat module. The names given in ``__all__`` are all considered public\nand are required to exist. If ``__all__`` is not defined, the set of\npublic names includes all names found in the module\'s namespace which\ndo not begin with an underscore character (``\'_\'``). ``__all__``\nshould contain the entire public API. It is intended to avoid\naccidentally exporting items that are not part of the API (such as\nlibrary modules which were imported and used within the module).\n\nThe ``from`` form with ``*`` may only occur in a module scope. If the\nwild card form of import --- ``import *`` --- is used in a function\nand the function contains or is a nested block with free variables,\nthe compiler will raise a ``SyntaxError``.\n\nWhen specifying what module to import you do not have to specify the\nabsolute name of the module. When a module or package is contained\nwithin another package it is possible to make a relative import within\nthe same top package without having to mention the package name. By\nusing leading dots in the specified module or package after ``from``\nyou can specify how high to traverse up the current package hierarchy\nwithout specifying exact names. One leading dot means the current\npackage where the module making the import exists. Two dots means up\none package level. Three dots is up two levels, etc. So if you execute\n``from . import mod`` from a module in the ``pkg`` package then you\nwill end up importing ``pkg.mod``. If you execute ``from ..subpkg2\nimport mod`` from within ``pkg.subpkg1`` you will import\n``pkg.subpkg2.mod``. The specification for relative imports is\ncontained within **PEP 328**.\n\n``importlib.import_module()`` is provided to support applications that\ndetermine which modules need to be loaded dynamically.\n\n\nFuture statements\n=================\n\nA *future statement* is a directive to the compiler that a particular\nmodule should be compiled using syntax or semantics that will be\navailable in a specified future release of Python. The future\nstatement is intended to ease migration to future versions of Python\nthat introduce incompatible changes to the language. It allows use of\nthe new features on a per-module basis before the release in which the\nfeature becomes standard.\n\n future_statement ::= "from" "__future__" "import" feature ["as" name]\n ("," feature ["as" name])*\n | "from" "__future__" "import" "(" feature ["as" name]\n ("," feature ["as" name])* [","] ")"\n feature ::= identifier\n name ::= identifier\n\nA future statement must appear near the top of the module. The only\nlines that can appear before a future statement are:\n\n* the module docstring (if any),\n\n* comments,\n\n* blank lines, and\n\n* other future statements.\n\nThe features recognized by Python 2.6 are ``unicode_literals``,\n``print_function``, ``absolute_import``, ``division``, ``generators``,\n``nested_scopes`` and ``with_statement``. ``generators``,\n``with_statement``, ``nested_scopes`` are redundant in Python version\n2.6 and above because they are always enabled.\n\nA future statement is recognized and treated specially at compile\ntime: Changes to the semantics of core constructs are often\nimplemented by generating different code. It may even be the case\nthat a new feature introduces new incompatible syntax (such as a new\nreserved word), in which case the compiler may need to parse the\nmodule differently. Such decisions cannot be pushed off until\nruntime.\n\nFor any given release, the compiler knows which feature names have\nbeen defined, and raises a compile-time error if a future statement\ncontains a feature not known to it.\n\nThe direct runtime semantics are the same as for any import statement:\nthere is a standard module ``__future__``, described later, and it\nwill be imported in the usual way at the time the future statement is\nexecuted.\n\nThe interesting runtime semantics depend on the specific feature\nenabled by the future statement.\n\nNote that there is nothing special about the statement:\n\n import __future__ [as name]\n\nThat is not a future statement; it\'s an ordinary import statement with\nno special semantics or syntax restrictions.\n\nCode compiled by an ``exec`` statement or calls to the built-in\nfunctions ``compile()`` and ``execfile()`` that occur in a module\n``M`` containing a future statement will, by default, use the new\nsyntax or semantics associated with the future statement. This can,\nstarting with Python 2.2 be controlled by optional arguments to\n``compile()`` --- see the documentation of that function for details.\n\nA future statement typed at an interactive interpreter prompt will\ntake effect for the rest of the interpreter session. If an\ninterpreter is started with the *-i* option, is passed a script name\nto execute, and the script includes a future statement, it will be in\neffect in the interactive session started after the script is\nexecuted.\n\nSee also:\n\n **PEP 236** - Back to the __future__\n The original proposal for the __future__ mechanism.\n', + 'import': '\nThe ``import`` statement\n************************\n\n import_stmt ::= "import" module ["as" name] ( "," module ["as" name] )*\n | "from" relative_module "import" identifier ["as" name]\n ( "," identifier ["as" name] )*\n | "from" relative_module "import" "(" identifier ["as" name]\n ( "," identifier ["as" name] )* [","] ")"\n | "from" module "import" "*"\n module ::= (identifier ".")* identifier\n relative_module ::= "."* module | "."+\n name ::= identifier\n\nImport statements are executed in two steps: (1) find a module, and\ninitialize it if necessary; (2) define a name or names in the local\nnamespace (of the scope where the ``import`` statement occurs). The\nstatement comes in two forms differing on whether it uses the ``from``\nkeyword. The first form (without ``from``) repeats these steps for\neach identifier in the list. The form with ``from`` performs step (1)\nonce, and then performs step (2) repeatedly.\n\nTo understand how step (1) occurs, one must first understand how\nPython handles hierarchical naming of modules. To help organize\nmodules and provide a hierarchy in naming, Python has a concept of\npackages. A package can contain other packages and modules while\nmodules cannot contain other modules or packages. From a file system\nperspective, packages are directories and modules are files. The\noriginal specification for packages is still available to read,\nalthough minor details have changed since the writing of that\ndocument.\n\nOnce the name of the module is known (unless otherwise specified, the\nterm "module" will refer to both packages and modules), searching for\nthe module or package can begin. The first place checked is\n``sys.modules``, the cache of all modules that have been imported\npreviously. If the module is found there then it is used in step (2)\nof import.\n\nIf the module is not found in the cache, then ``sys.meta_path`` is\nsearched (the specification for ``sys.meta_path`` can be found in\n**PEP 302**). The object is a list of *finder* objects which are\nqueried in order as to whether they know how to load the module by\ncalling their ``find_module()`` method with the name of the module. If\nthe module happens to be contained within a package (as denoted by the\nexistence of a dot in the name), then a second argument to\n``find_module()`` is given as the value of the ``__path__`` attribute\nfrom the parent package (everything up to the last dot in the name of\nthe module being imported). If a finder can find the module it returns\na *loader* (discussed later) or returns ``None``.\n\nIf none of the finders on ``sys.meta_path`` are able to find the\nmodule then some implicitly defined finders are queried.\nImplementations of Python vary in what implicit meta path finders are\ndefined. The one they all do define, though, is one that handles\n``sys.path_hooks``, ``sys.path_importer_cache``, and ``sys.path``.\n\nThe implicit finder searches for the requested module in the "paths"\nspecified in one of two places ("paths" do not have to be file system\npaths). If the module being imported is supposed to be contained\nwithin a package then the second argument passed to ``find_module()``,\n``__path__`` on the parent package, is used as the source of paths. If\nthe module is not contained in a package then ``sys.path`` is used as\nthe source of paths.\n\nOnce the source of paths is chosen it is iterated over to find a\nfinder that can handle that path. The dict at\n``sys.path_importer_cache`` caches finders for paths and is checked\nfor a finder. If the path does not have a finder cached then\n``sys.path_hooks`` is searched by calling each object in the list with\na single argument of the path, returning a finder or raises\n``ImportError``. If a finder is returned then it is cached in\n``sys.path_importer_cache`` and then used for that path entry. If no\nfinder can be found but the path exists then a value of ``None`` is\nstored in ``sys.path_importer_cache`` to signify that an implicit,\nfile-based finder that handles modules stored as individual files\nshould be used for that path. If the path does not exist then a finder\nwhich always returns ``None`` is placed in the cache for the path.\n\nIf no finder can find the module then ``ImportError`` is raised.\nOtherwise some finder returned a loader whose ``load_module()`` method\nis called with the name of the module to load (see **PEP 302** for the\noriginal definition of loaders). A loader has several responsibilities\nto perform on a module it loads. First, if the module already exists\nin ``sys.modules`` (a possibility if the loader is called outside of\nthe import machinery) then it is to use that module for initialization\nand not a new module. But if the module does not exist in\n``sys.modules`` then it is to be added to that dict before\ninitialization begins. If an error occurs during loading of the module\nand it was added to ``sys.modules`` it is to be removed from the dict.\nIf an error occurs but the module was already in ``sys.modules`` it is\nleft in the dict.\n\nThe loader must set several attributes on the module. ``__name__`` is\nto be set to the name of the module. ``__file__`` is to be the "path"\nto the file unless the module is built-in (and thus listed in\n``sys.builtin_module_names``) in which case the attribute is not set.\nIf what is being imported is a package then ``__path__`` is to be set\nto a list of paths to be searched when looking for modules and\npackages contained within the package being imported. ``__package__``\nis optional but should be set to the name of package that contains the\nmodule or package (the empty string is used for module not contained\nin a package). ``__loader__`` is also optional but should be set to\nthe loader object that is loading the module.\n\nIf an error occurs during loading then the loader raises\n``ImportError`` if some other exception is not already being\npropagated. Otherwise the loader returns the module that was loaded\nand initialized.\n\nWhen step (1) finishes without raising an exception, step (2) can\nbegin.\n\nThe first form of ``import`` statement binds the module name in the\nlocal namespace to the module object, and then goes on to import the\nnext identifier, if any. If the module name is followed by ``as``,\nthe name following ``as`` is used as the local name for the module.\n\nThe ``from`` form does not bind the module name: it goes through the\nlist of identifiers, looks each one of them up in the module found in\nstep (1), and binds the name in the local namespace to the object thus\nfound. As with the first form of ``import``, an alternate local name\ncan be supplied by specifying "``as`` localname". If a name is not\nfound, ``ImportError`` is raised. If the list of identifiers is\nreplaced by a star (``\'*\'``), all public names defined in the module\nare bound in the local namespace of the ``import`` statement..\n\nThe *public names* defined by a module are determined by checking the\nmodule\'s namespace for a variable named ``__all__``; if defined, it\nmust be a sequence of strings which are names defined or imported by\nthat module. The names given in ``__all__`` are all considered public\nand are required to exist. If ``__all__`` is not defined, the set of\npublic names includes all names found in the module\'s namespace which\ndo not begin with an underscore character (``\'_\'``). ``__all__``\nshould contain the entire public API. It is intended to avoid\naccidentally exporting items that are not part of the API (such as\nlibrary modules which were imported and used within the module).\n\nThe ``from`` form with ``*`` may only occur in a module scope. If the\nwild card form of import --- ``import *`` --- is used in a function\nand the function contains or is a nested block with free variables,\nthe compiler will raise a ``SyntaxError``.\n\nWhen specifying what module to import you do not have to specify the\nabsolute name of the module. When a module or package is contained\nwithin another package it is possible to make a relative import within\nthe same top package without having to mention the package name. By\nusing leading dots in the specified module or package after ``from``\nyou can specify how high to traverse up the current package hierarchy\nwithout specifying exact names. One leading dot means the current\npackage where the module making the import exists. Two dots means up\none package level. Three dots is up two levels, etc. So if you execute\n``from . import mod`` from a module in the ``pkg`` package then you\nwill end up importing ``pkg.mod``. If you execute ``from ..subpkg2\nimport mod`` from within ``pkg.subpkg1`` you will import\n``pkg.subpkg2.mod``. The specification for relative imports is\ncontained within **PEP 328**.\n\n``importlib.import_module()`` is provided to support applications that\ndetermine which modules need to be loaded dynamically.\n\n\nFuture statements\n=================\n\nA *future statement* is a directive to the compiler that a particular\nmodule should be compiled using syntax or semantics that will be\navailable in a specified future release of Python. The future\nstatement is intended to ease migration to future versions of Python\nthat introduce incompatible changes to the language. It allows use of\nthe new features on a per-module basis before the release in which the\nfeature becomes standard.\n\n future_statement ::= "from" "__future__" "import" feature ["as" name]\n ("," feature ["as" name])*\n | "from" "__future__" "import" "(" feature ["as" name]\n ("," feature ["as" name])* [","] ")"\n feature ::= identifier\n name ::= identifier\n\nA future statement must appear near the top of the module. The only\nlines that can appear before a future statement are:\n\n* the module docstring (if any),\n\n* comments,\n\n* blank lines, and\n\n* other future statements.\n\nThe features recognized by Python 2.6 are ``unicode_literals``,\n``print_function``, ``absolute_import``, ``division``, ``generators``,\n``nested_scopes`` and ``with_statement``. ``generators``,\n``with_statement``, ``nested_scopes`` are redundant in Python version\n2.6 and above because they are always enabled.\n\nA future statement is recognized and treated specially at compile\ntime: Changes to the semantics of core constructs are often\nimplemented by generating different code. It may even be the case\nthat a new feature introduces new incompatible syntax (such as a new\nreserved word), in which case the compiler may need to parse the\nmodule differently. Such decisions cannot be pushed off until\nruntime.\n\nFor any given release, the compiler knows which feature names have\nbeen defined, and raises a compile-time error if a future statement\ncontains a feature not known to it.\n\nThe direct runtime semantics are the same as for any import statement:\nthere is a standard module ``__future__``, described later, and it\nwill be imported in the usual way at the time the future statement is\nexecuted.\n\nThe interesting runtime semantics depend on the specific feature\nenabled by the future statement.\n\nNote that there is nothing special about the statement:\n\n import __future__ [as name]\n\nThat is not a future statement; it\'s an ordinary import statement with\nno special semantics or syntax restrictions.\n\nCode compiled by an ``exec`` statement or calls to the built-in\nfunctions ``compile()`` and ``execfile()`` that occur in a module\n``M`` containing a future statement will, by default, use the new\nsyntax or semantics associated with the future statement. This can,\nstarting with Python 2.2 be controlled by optional arguments to\n``compile()`` --- see the documentation of that function for details.\n\nA future statement typed at an interactive interpreter prompt will\ntake effect for the rest of the interpreter session. If an\ninterpreter is started with the *-i* option, is passed a script name\nto execute, and the script includes a future statement, it will be in\neffect in the interactive session started after the script is\nexecuted.\n\nSee also:\n\n **PEP 236** - Back to the __future__\n The original proposal for the __future__ mechanism.\n', 'in': '\nComparisons\n***********\n\nUnlike C, all comparison operations in Python have the same priority,\nwhich is lower than that of any arithmetic, shifting or bitwise\noperation. Also unlike C, expressions like ``a < b < c`` have the\ninterpretation that is conventional in mathematics:\n\n comparison ::= or_expr ( comp_operator or_expr )*\n comp_operator ::= "<" | ">" | "==" | ">=" | "<=" | "<>" | "!="\n | "is" ["not"] | ["not"] "in"\n\nComparisons yield boolean values: ``True`` or ``False``.\n\nComparisons can be chained arbitrarily, e.g., ``x < y <= z`` is\nequivalent to ``x < y and y <= z``, except that ``y`` is evaluated\nonly once (but in both cases ``z`` is not evaluated at all when ``x <\ny`` is found to be false).\n\nFormally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*,\n*op2*, ..., *opN* are comparison operators, then ``a op1 b op2 c ... y\nopN z`` is equivalent to ``a op1 b and b op2 c and ... y opN z``,\nexcept that each expression is evaluated at most once.\n\nNote that ``a op1 b op2 c`` doesn\'t imply any kind of comparison\nbetween *a* and *c*, so that, e.g., ``x < y > z`` is perfectly legal\n(though perhaps not pretty).\n\nThe forms ``<>`` and ``!=`` are equivalent; for consistency with C,\n``!=`` is preferred; where ``!=`` is mentioned below ``<>`` is also\naccepted. The ``<>`` spelling is considered obsolescent.\n\nThe operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare\nthe values of two objects. The objects need not have the same type.\nIf both are numbers, they are converted to a common type. Otherwise,\nobjects of different types *always* compare unequal, and are ordered\nconsistently but arbitrarily. You can control comparison behavior of\nobjects of non-built-in types by defining a ``__cmp__`` method or rich\ncomparison methods like ``__gt__``, described in section *Special\nmethod names*.\n\n(This unusual definition of comparison was used to simplify the\ndefinition of operations like sorting and the ``in`` and ``not in``\noperators. In the future, the comparison rules for objects of\ndifferent types are likely to change.)\n\nComparison of objects of the same type depends on the type:\n\n* Numbers are compared arithmetically.\n\n* Strings are compared lexicographically using the numeric equivalents\n (the result of the built-in function ``ord()``) of their characters.\n Unicode and 8-bit strings are fully interoperable in this behavior.\n [4]\n\n* Tuples and lists are compared lexicographically using comparison of\n corresponding elements. This means that to compare equal, each\n element must compare equal and the two sequences must be of the same\n type and have the same length.\n\n If not equal, the sequences are ordered the same as their first\n differing elements. For example, ``cmp([1,2,x], [1,2,y])`` returns\n the same as ``cmp(x,y)``. If the corresponding element does not\n exist, the shorter sequence is ordered first (for example, ``[1,2] <\n [1,2,3]``).\n\n* Mappings (dictionaries) compare equal if and only if their sorted\n (key, value) lists compare equal. [5] Outcomes other than equality\n are resolved consistently, but are not otherwise defined. [6]\n\n* Most other objects of built-in types compare unequal unless they are\n the same object; the choice whether one object is considered smaller\n or larger than another one is made arbitrarily but consistently\n within one execution of a program.\n\nThe operators ``in`` and ``not in`` test for collection membership.\n``x in s`` evaluates to true if *x* is a member of the collection *s*,\nand false otherwise. ``x not in s`` returns the negation of ``x in\ns``. The collection membership test has traditionally been bound to\nsequences; an object is a member of a collection if the collection is\na sequence and contains an element equal to that object. However, it\nmake sense for many other object types to support membership tests\nwithout being a sequence. In particular, dictionaries (for keys) and\nsets support membership testing.\n\nFor the list and tuple types, ``x in y`` is true if and only if there\nexists an index *i* such that ``x == y[i]`` is true.\n\nFor the Unicode and string types, ``x in y`` is true if and only if\n*x* is a substring of *y*. An equivalent test is ``y.find(x) != -1``.\nNote, *x* and *y* need not be the same type; consequently, ``u\'ab\' in\n\'abc\'`` will return ``True``. Empty strings are always considered to\nbe a substring of any other string, so ``"" in "abc"`` will return\n``True``.\n\nChanged in version 2.3: Previously, *x* was required to be a string of\nlength ``1``.\n\nFor user-defined classes which define the ``__contains__()`` method,\n``x in y`` is true if and only if ``y.__contains__(x)`` is true.\n\nFor user-defined classes which do not define ``__contains__()`` but do\ndefine ``__iter__()``, ``x in y`` is true if some value ``z`` with ``x\n== z`` is produced while iterating over ``y``. If an exception is\nraised during the iteration, it is as if ``in`` raised that exception.\n\nLastly, the old-style iteration protocol is tried: if a class defines\n``__getitem__()``, ``x in y`` is true if and only if there is a non-\nnegative integer index *i* such that ``x == y[i]``, and all lower\ninteger indices do not raise ``IndexError`` exception. (If any other\nexception is raised, it is as if ``in`` raised that exception).\n\nThe operator ``not in`` is defined to have the inverse true value of\n``in``.\n\nThe operators ``is`` and ``is not`` test for object identity: ``x is\ny`` is true if and only if *x* and *y* are the same object. ``x is\nnot y`` yields the inverse truth value. [7]\n', 'integers': '\nInteger and long integer literals\n*********************************\n\nInteger and long integer literals are described by the following\nlexical definitions:\n\n longinteger ::= integer ("l" | "L")\n integer ::= decimalinteger | octinteger | hexinteger | bininteger\n decimalinteger ::= nonzerodigit digit* | "0"\n octinteger ::= "0" ("o" | "O") octdigit+ | "0" octdigit+\n hexinteger ::= "0" ("x" | "X") hexdigit+\n bininteger ::= "0" ("b" | "B") bindigit+\n nonzerodigit ::= "1"..."9"\n octdigit ::= "0"..."7"\n bindigit ::= "0" | "1"\n hexdigit ::= digit | "a"..."f" | "A"..."F"\n\nAlthough both lower case ``\'l\'`` and upper case ``\'L\'`` are allowed as\nsuffix for long integers, it is strongly recommended to always use\n``\'L\'``, since the letter ``\'l\'`` looks too much like the digit\n``\'1\'``.\n\nPlain integer literals that are above the largest representable plain\ninteger (e.g., 2147483647 when using 32-bit arithmetic) are accepted\nas if they were long integers instead. [1] There is no limit for long\ninteger literals apart from what can be stored in available memory.\n\nSome examples of plain integer literals (first row) and long integer\nliterals (second and third rows):\n\n 7 2147483647 0177\n 3L 79228162514264337593543950336L 0377L 0x100000000L\n 79228162514264337593543950336 0xdeadbeef\n', 'lambda': '\nLambdas\n*******\n\n lambda_form ::= "lambda" [parameter_list]: expression\n old_lambda_form ::= "lambda" [parameter_list]: old_expression\n\nLambda forms (lambda expressions) have the same syntactic position as\nexpressions. They are a shorthand to create anonymous functions; the\nexpression ``lambda arguments: expression`` yields a function object.\nThe unnamed object behaves like a function object defined with\n\n def name(arguments):\n return expression\n\nSee section *Function definitions* for the syntax of parameter lists.\nNote that functions created with lambda forms cannot contain\nstatements.\n', @@ -49,7 +49,7 @@ 'numbers': "\nNumeric literals\n****************\n\nThere are four types of numeric literals: plain integers, long\nintegers, floating point numbers, and imaginary numbers. There are no\ncomplex literals (complex numbers can be formed by adding a real\nnumber and an imaginary number).\n\nNote that numeric literals do not include a sign; a phrase like ``-1``\nis actually an expression composed of the unary operator '``-``' and\nthe literal ``1``.\n", 'numeric-types': '\nEmulating numeric types\n***********************\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``//``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For\n instance, to evaluate the expression ``x + y``, where *x* is an\n instance of a class that has an ``__add__()`` method,\n ``x.__add__(y)`` is called. The ``__divmod__()`` method should be\n the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n should not be related to ``__truediv__()`` (described below). Note\n that ``__pow__()`` should be defined to accept an optional third\n argument if the ternary version of the built-in ``pow()`` function\n is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return ``NotImplemented``.\n\nobject.__div__(self, other)\nobject.__truediv__(self, other)\n\n The division operator (``/``) is implemented by these methods. The\n ``__truediv__()`` method is used when ``__future__.division`` is in\n effect, otherwise ``__div__()`` is used. If only one of these two\n methods is defined, the object will not support division in the\n alternate context; ``TypeError`` will be raised instead.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rdiv__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with\n reflected (swapped) operands. These functions are only called if\n the left operand does not support the corresponding operation and\n the operands are of different types. [2] For instance, to evaluate\n the expression ``x - y``, where *y* is an instance of a class that\n has an ``__rsub__()`` method, ``y.__rsub__(x)`` is called if\n ``x.__sub__(y)`` returns *NotImplemented*.\n\n Note that ternary ``pow()`` will not try calling ``__rpow__()``\n (the coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left operand\'s\n type and that subclass provides the reflected method for the\n operation, this method will be called before the left operand\'s\n non-reflected method. This behavior allows subclasses to\n override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__idiv__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods\n should attempt to do the operation in-place (modifying *self*) and\n return the result (which could be, but does not have to be,\n *self*). If a specific method is not defined, the augmented\n assignment falls back to the normal methods. For instance, to\n execute the statement ``x += y``, where *x* is an instance of a\n class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n called. If *x* is an instance of a class that does not define a\n ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations (``-``, ``+``,\n ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__long__(self)\nobject.__float__(self)\n\n Called to implement the built-in functions ``complex()``,\n ``int()``, ``long()``, and ``float()``. Should return a value of\n the appropriate type.\n\nobject.__oct__(self)\nobject.__hex__(self)\n\n Called to implement the built-in functions ``oct()`` and ``hex()``.\n Should return a string value.\n\nobject.__index__(self)\n\n Called to implement ``operator.index()``. Also called whenever\n Python needs an integer object (such as in slicing). Must return\n an integer (int or long).\n\n New in version 2.5.\n\nobject.__coerce__(self, other)\n\n Called to implement "mixed-mode" numeric arithmetic. Should either\n return a 2-tuple containing *self* and *other* converted to a\n common numeric type, or ``None`` if conversion is impossible. When\n the common type would be the type of ``other``, it is sufficient to\n return ``None``, since the interpreter will also ask the other\n object to attempt a coercion (but sometimes, if the implementation\n of the other type cannot be changed, it is useful to do the\n conversion to the other type here). A return value of\n ``NotImplemented`` is equivalent to returning ``None``.\n', 'objects': '\nObjects, values and types\n*************************\n\n*Objects* are Python\'s abstraction for data. All data in a Python\nprogram is represented by objects or by relations between objects. (In\na sense, and in conformance to Von Neumann\'s model of a "stored\nprogram computer," code is also represented by objects.)\n\nEvery object has an identity, a type and a value. An object\'s\n*identity* never changes once it has been created; you may think of it\nas the object\'s address in memory. The \'``is``\' operator compares the\nidentity of two objects; the ``id()`` function returns an integer\nrepresenting its identity (currently implemented as its address). An\nobject\'s *type* is also unchangeable. [1] An object\'s type determines\nthe operations that the object supports (e.g., "does it have a\nlength?") and also defines the possible values for objects of that\ntype. The ``type()`` function returns an object\'s type (which is an\nobject itself). The *value* of some objects can change. Objects\nwhose value can change are said to be *mutable*; objects whose value\nis unchangeable once they are created are called *immutable*. (The\nvalue of an immutable container object that contains a reference to a\nmutable object can change when the latter\'s value is changed; however\nthe container is still considered immutable, because the collection of\nobjects it contains cannot be changed. So, immutability is not\nstrictly the same as having an unchangeable value, it is more subtle.)\nAn object\'s mutability is determined by its type; for instance,\nnumbers, strings and tuples are immutable, while dictionaries and\nlists are mutable.\n\nObjects are never explicitly destroyed; however, when they become\nunreachable they may be garbage-collected. An implementation is\nallowed to postpone garbage collection or omit it altogether --- it is\na matter of implementation quality how garbage collection is\nimplemented, as long as no objects are collected that are still\nreachable.\n\n**CPython implementation detail:** CPython currently uses a reference-\ncounting scheme with (optional) delayed detection of cyclically linked\ngarbage, which collects most objects as soon as they become\nunreachable, but is not guaranteed to collect garbage containing\ncircular references. See the documentation of the ``gc`` module for\ninformation on controlling the collection of cyclic garbage. Other\nimplementations act differently and CPython may change. Do not depend\non immediate finalization of objects when they become unreachable (ex:\nalways close files).\n\nNote that the use of the implementation\'s tracing or debugging\nfacilities may keep objects alive that would normally be collectable.\nAlso note that catching an exception with a \'``try``...``except``\'\nstatement may keep objects alive.\n\nSome objects contain references to "external" resources such as open\nfiles or windows. It is understood that these resources are freed\nwhen the object is garbage-collected, but since garbage collection is\nnot guaranteed to happen, such objects also provide an explicit way to\nrelease the external resource, usually a ``close()`` method. Programs\nare strongly recommended to explicitly close such objects. The\n\'``try``...``finally``\' statement provides a convenient way to do\nthis.\n\nSome objects contain references to other objects; these are called\n*containers*. Examples of containers are tuples, lists and\ndictionaries. The references are part of a container\'s value. In\nmost cases, when we talk about the value of a container, we imply the\nvalues, not the identities of the contained objects; however, when we\ntalk about the mutability of a container, only the identities of the\nimmediately contained objects are implied. So, if an immutable\ncontainer (like a tuple) contains a reference to a mutable object, its\nvalue changes if that mutable object is changed.\n\nTypes affect almost all aspects of object behavior. Even the\nimportance of object identity is affected in some sense: for immutable\ntypes, operations that compute new values may actually return a\nreference to any existing object with the same type and value, while\nfor mutable objects this is not allowed. E.g., after ``a = 1; b =\n1``, ``a`` and ``b`` may or may not refer to the same object with the\nvalue one, depending on the implementation, but after ``c = []; d =\n[]``, ``c`` and ``d`` are guaranteed to refer to two different,\nunique, newly created empty lists. (Note that ``c = d = []`` assigns\nthe same object to both ``c`` and ``d``.)\n', - 'operator-summary': '\nOperator precedence\n*******************\n\nThe following table summarizes the operator precedences in Python,\nfrom lowest precedence (least binding) to highest precedence (most\nbinding). Operators in the same box have the same precedence. Unless\nthe syntax is explicitly given, operators are binary. Operators in\nthe same box group left to right (except for comparisons, including\ntests, which all have the same precedence and chain from left to right\n--- see section *Comparisons* --- and exponentiation, which groups\nfrom right to left).\n\n+-------------------------------------------------+---------------------------------------+\n| Operator | Description |\n+=================================================+=======================================+\n| ``lambda`` | Lambda expression |\n+-------------------------------------------------+---------------------------------------+\n| ``if`` -- ``else`` | Conditional expression |\n+-------------------------------------------------+---------------------------------------+\n| ``or`` | Boolean OR |\n+-------------------------------------------------+---------------------------------------+\n| ``and`` | Boolean AND |\n+-------------------------------------------------+---------------------------------------+\n| ``not`` ``x`` | Boolean NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``in``, ``not in``, ``is``, ``is not``, ``<``, | Comparisons, including membership |\n| ``<=``, ``>``, ``>=``, ``<>``, ``!=``, ``==`` | tests and identity tests, |\n+-------------------------------------------------+---------------------------------------+\n| ``|`` | Bitwise OR |\n+-------------------------------------------------+---------------------------------------+\n| ``^`` | Bitwise XOR |\n+-------------------------------------------------+---------------------------------------+\n| ``&`` | Bitwise AND |\n+-------------------------------------------------+---------------------------------------+\n| ``<<``, ``>>`` | Shifts |\n+-------------------------------------------------+---------------------------------------+\n| ``+``, ``-`` | Addition and subtraction |\n+-------------------------------------------------+---------------------------------------+\n| ``*``, ``/``, ``//``, ``%`` | Multiplication, division, remainder |\n| | [8] |\n+-------------------------------------------------+---------------------------------------+\n| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``**`` | Exponentiation [9] |\n+-------------------------------------------------+---------------------------------------+\n| ``x[index]``, ``x[index:index]``, | Subscription, slicing, call, |\n| ``x(arguments...)``, ``x.attribute`` | attribute reference |\n+-------------------------------------------------+---------------------------------------+\n| ``(expressions...)``, ``[expressions...]``, | Binding or tuple display, list |\n| ``{key: value...}``, ```expressions...``` | display, dictionary display, string |\n| | conversion |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] In Python 2.3 and later releases, a list comprehension "leaks" the\n control variables of each ``for`` it contains into the containing\n scope. However, this behavior is deprecated, and relying on it\n will not work in Python 3.\n\n[2] While ``abs(x%y) < abs(y)`` is true mathematically, for floats it\n may not be true numerically due to roundoff. For example, and\n assuming a platform on which a Python float is an IEEE 754 double-\n precision number, in order that ``-1e-100 % 1e100`` have the same\n sign as ``1e100``, the computed result is ``-1e-100 + 1e100``,\n which is numerically exactly equal to ``1e100``. The function\n ``math.fmod()`` returns a result whose sign matches the sign of\n the first argument instead, and so returns ``-1e-100`` in this\n case. Which approach is more appropriate depends on the\n application.\n\n[3] If x is very close to an exact integer multiple of y, it\'s\n possible for ``floor(x/y)`` to be one larger than ``(x-x%y)/y``\n due to rounding. In such cases, Python returns the latter result,\n in order to preserve that ``divmod(x,y)[0] * y + x % y`` be very\n close to ``x``.\n\n[4] While comparisons between unicode strings make sense at the byte\n level, they may be counter-intuitive to users. For example, the\n strings ``u"\\u00C7"`` and ``u"\\u0043\\u0327"`` compare differently,\n even though they both represent the same unicode character (LATIN\n CAPITAL LETTER C WITH CEDILLA). To compare strings in a human\n recognizable way, compare using ``unicodedata.normalize()``.\n\n[5] The implementation computes this efficiently, without constructing\n lists or sorting.\n\n[6] Earlier versions of Python used lexicographic comparison of the\n sorted (key, value) lists, but this was very expensive for the\n common case of comparing for equality. An even earlier version of\n Python compared dictionaries by identity only, but this caused\n surprises because people expected to be able to test a dictionary\n for emptiness by comparing it to ``{}``.\n\n[7] Due to automatic garbage-collection, free lists, and the dynamic\n nature of descriptors, you may notice seemingly unusual behaviour\n in certain uses of the ``is`` operator, like those involving\n comparisons between instance methods, or constants. Check their\n documentation for more info.\n\n[8] The ``%`` operator is also used for string formatting; the same\n precedence applies.\n\n[9] The power operator ``**`` binds less tightly than an arithmetic or\n bitwise unary operator on its right, that is, ``2**-1`` is\n ``0.5``.\n', + 'operator-summary': '\nOperator precedence\n*******************\n\nThe following table summarizes the operator precedences in Python,\nfrom lowest precedence (least binding) to highest precedence (most\nbinding). Operators in the same box have the same precedence. Unless\nthe syntax is explicitly given, operators are binary. Operators in\nthe same box group left to right (except for comparisons, including\ntests, which all have the same precedence and chain from left to right\n--- see section *Comparisons* --- and exponentiation, which groups\nfrom right to left).\n\n+-------------------------------------------------+---------------------------------------+\n| Operator | Description |\n+=================================================+=======================================+\n| ``lambda`` | Lambda expression |\n+-------------------------------------------------+---------------------------------------+\n| ``if`` -- ``else`` | Conditional expression |\n+-------------------------------------------------+---------------------------------------+\n| ``or`` | Boolean OR |\n+-------------------------------------------------+---------------------------------------+\n| ``and`` | Boolean AND |\n+-------------------------------------------------+---------------------------------------+\n| ``not`` ``x`` | Boolean NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``in``, ``not in``, ``is``, ``is not``, ``<``, | Comparisons, including membership |\n| ``<=``, ``>``, ``>=``, ``<>``, ``!=``, ``==`` | tests and identity tests |\n+-------------------------------------------------+---------------------------------------+\n| ``|`` | Bitwise OR |\n+-------------------------------------------------+---------------------------------------+\n| ``^`` | Bitwise XOR |\n+-------------------------------------------------+---------------------------------------+\n| ``&`` | Bitwise AND |\n+-------------------------------------------------+---------------------------------------+\n| ``<<``, ``>>`` | Shifts |\n+-------------------------------------------------+---------------------------------------+\n| ``+``, ``-`` | Addition and subtraction |\n+-------------------------------------------------+---------------------------------------+\n| ``*``, ``/``, ``//``, ``%`` | Multiplication, division, remainder |\n| | [8] |\n+-------------------------------------------------+---------------------------------------+\n| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT |\n+-------------------------------------------------+---------------------------------------+\n| ``**`` | Exponentiation [9] |\n+-------------------------------------------------+---------------------------------------+\n| ``x[index]``, ``x[index:index]``, | Subscription, slicing, call, |\n| ``x(arguments...)``, ``x.attribute`` | attribute reference |\n+-------------------------------------------------+---------------------------------------+\n| ``(expressions...)``, ``[expressions...]``, | Binding or tuple display, list |\n| ``{key: value...}``, ```expressions...``` | display, dictionary display, string |\n| | conversion |\n+-------------------------------------------------+---------------------------------------+\n\n-[ Footnotes ]-\n\n[1] In Python 2.3 and later releases, a list comprehension "leaks" the\n control variables of each ``for`` it contains into the containing\n scope. However, this behavior is deprecated, and relying on it\n will not work in Python 3.\n\n[2] While ``abs(x%y) < abs(y)`` is true mathematically, for floats it\n may not be true numerically due to roundoff. For example, and\n assuming a platform on which a Python float is an IEEE 754 double-\n precision number, in order that ``-1e-100 % 1e100`` have the same\n sign as ``1e100``, the computed result is ``-1e-100 + 1e100``,\n which is numerically exactly equal to ``1e100``. The function\n ``math.fmod()`` returns a result whose sign matches the sign of\n the first argument instead, and so returns ``-1e-100`` in this\n case. Which approach is more appropriate depends on the\n application.\n\n[3] If x is very close to an exact integer multiple of y, it\'s\n possible for ``floor(x/y)`` to be one larger than ``(x-x%y)/y``\n due to rounding. In such cases, Python returns the latter result,\n in order to preserve that ``divmod(x,y)[0] * y + x % y`` be very\n close to ``x``.\n\n[4] While comparisons between unicode strings make sense at the byte\n level, they may be counter-intuitive to users. For example, the\n strings ``u"\\u00C7"`` and ``u"\\u0043\\u0327"`` compare differently,\n even though they both represent the same unicode character (LATIN\n CAPITAL LETTER C WITH CEDILLA). To compare strings in a human\n recognizable way, compare using ``unicodedata.normalize()``.\n\n[5] The implementation computes this efficiently, without constructing\n lists or sorting.\n\n[6] Earlier versions of Python used lexicographic comparison of the\n sorted (key, value) lists, but this was very expensive for the\n common case of comparing for equality. An even earlier version of\n Python compared dictionaries by identity only, but this caused\n surprises because people expected to be able to test a dictionary\n for emptiness by comparing it to ``{}``.\n\n[7] Due to automatic garbage-collection, free lists, and the dynamic\n nature of descriptors, you may notice seemingly unusual behaviour\n in certain uses of the ``is`` operator, like those involving\n comparisons between instance methods, or constants. Check their\n documentation for more info.\n\n[8] The ``%`` operator is also used for string formatting; the same\n precedence applies.\n\n[9] The power operator ``**`` binds less tightly than an arithmetic or\n bitwise unary operator on its right, that is, ``2**-1`` is\n ``0.5``.\n', 'pass': '\nThe ``pass`` statement\n**********************\n\n pass_stmt ::= "pass"\n\n``pass`` is a null operation --- when it is executed, nothing happens.\nIt is useful as a placeholder when a statement is required\nsyntactically, but no code needs to be executed, for example:\n\n def f(arg): pass # a function that does nothing (yet)\n\n class C: pass # a class with no methods (yet)\n', 'power': '\nThe power operator\n******************\n\nThe power operator binds more tightly than unary operators on its\nleft; it binds less tightly than unary operators on its right. The\nsyntax is:\n\n power ::= primary ["**" u_expr]\n\nThus, in an unparenthesized sequence of power and unary operators, the\noperators are evaluated from right to left (this does not constrain\nthe evaluation order for the operands): ``-1**2`` results in ``-1``.\n\nThe power operator has the same semantics as the built-in ``pow()``\nfunction, when called with two arguments: it yields its left argument\nraised to the power of its right argument. The numeric arguments are\nfirst converted to a common type. The result type is that of the\narguments after coercion.\n\nWith mixed operand types, the coercion rules for binary arithmetic\noperators apply. For int and long int operands, the result has the\nsame type as the operands (after coercion) unless the second argument\nis negative; in that case, all arguments are converted to float and a\nfloat result is delivered. For example, ``10**2`` returns ``100``, but\n``10**-2`` returns ``0.01``. (This last feature was added in Python\n2.2. In Python 2.1 and before, if both arguments were of integer types\nand the second argument was negative, an exception was raised).\n\nRaising ``0.0`` to a negative power results in a\n``ZeroDivisionError``. Raising a negative number to a fractional power\nresults in a ``ValueError``.\n', 'raise': '\nThe ``raise`` statement\n***********************\n\n raise_stmt ::= "raise" [expression ["," expression ["," expression]]]\n\nIf no expressions are present, ``raise`` re-raises the last exception\nthat was active in the current scope. If no exception is active in\nthe current scope, a ``TypeError`` exception is raised indicating that\nthis is an error (if running under IDLE, a ``Queue.Empty`` exception\nis raised instead).\n\nOtherwise, ``raise`` evaluates the expressions to get three objects,\nusing ``None`` as the value of omitted expressions. The first two\nobjects are used to determine the *type* and *value* of the exception.\n\nIf the first object is an instance, the type of the exception is the\nclass of the instance, the instance itself is the value, and the\nsecond object must be ``None``.\n\nIf the first object is a class, it becomes the type of the exception.\nThe second object is used to determine the exception value: If it is\nan instance of the class, the instance becomes the exception value. If\nthe second object is a tuple, it is used as the argument list for the\nclass constructor; if it is ``None``, an empty argument list is used,\nand any other object is treated as a single argument to the\nconstructor. The instance so created by calling the constructor is\nused as the exception value.\n\nIf a third object is present and not ``None``, it must be a traceback\nobject (see section *The standard type hierarchy*), and it is\nsubstituted instead of the current location as the place where the\nexception occurred. If the third object is present and not a\ntraceback object or ``None``, a ``TypeError`` exception is raised.\nThe three-expression form of ``raise`` is useful to re-raise an\nexception transparently in an except clause, but ``raise`` with no\nexpressions should be preferred if the exception to be re-raised was\nthe most recently active exception in the current scope.\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information about handling exceptions is in section\n*The try statement*.\n', @@ -59,19 +59,19 @@ 'slicings': '\nSlicings\n********\n\nA slicing selects a range of items in a sequence object (e.g., a\nstring, tuple or list). Slicings may be used as expressions or as\ntargets in assignment or ``del`` statements. The syntax for a\nslicing:\n\n slicing ::= simple_slicing | extended_slicing\n simple_slicing ::= primary "[" short_slice "]"\n extended_slicing ::= primary "[" slice_list "]"\n slice_list ::= slice_item ("," slice_item)* [","]\n slice_item ::= expression | proper_slice | ellipsis\n proper_slice ::= short_slice | long_slice\n short_slice ::= [lower_bound] ":" [upper_bound]\n long_slice ::= short_slice ":" [stride]\n lower_bound ::= expression\n upper_bound ::= expression\n stride ::= expression\n ellipsis ::= "..."\n\nThere is ambiguity in the formal syntax here: anything that looks like\nan expression list also looks like a slice list, so any subscription\ncan be interpreted as a slicing. Rather than further complicating the\nsyntax, this is disambiguated by defining that in this case the\ninterpretation as a subscription takes priority over the\ninterpretation as a slicing (this is the case if the slice list\ncontains no proper slice nor ellipses). Similarly, when the slice\nlist has exactly one short slice and no trailing comma, the\ninterpretation as a simple slicing takes priority over that as an\nextended slicing.\n\nThe semantics for a simple slicing are as follows. The primary must\nevaluate to a sequence object. The lower and upper bound expressions,\nif present, must evaluate to plain integers; defaults are zero and the\n``sys.maxint``, respectively. If either bound is negative, the\nsequence\'s length is added to it. The slicing now selects all items\nwith index *k* such that ``i <= k < j`` where *i* and *j* are the\nspecified lower and upper bounds. This may be an empty sequence. It\nis not an error if *i* or *j* lie outside the range of valid indexes\n(such items don\'t exist so they aren\'t selected).\n\nThe semantics for an extended slicing are as follows. The primary\nmust evaluate to a mapping object, and it is indexed with a key that\nis constructed from the slice list, as follows. If the slice list\ncontains at least one comma, the key is a tuple containing the\nconversion of the slice items; otherwise, the conversion of the lone\nslice item is the key. The conversion of a slice item that is an\nexpression is that expression. The conversion of an ellipsis slice\nitem is the built-in ``Ellipsis`` object. The conversion of a proper\nslice is a slice object (see section *The standard type hierarchy*)\nwhose ``start``, ``stop`` and ``step`` attributes are the values of\nthe expressions given as lower bound, upper bound and stride,\nrespectively, substituting ``None`` for missing expressions.\n', 'specialattrs': '\nSpecial Attributes\n******************\n\nThe implementation adds a few special read-only attributes to several\nobject types, where they are relevant. Some of these are not reported\nby the ``dir()`` built-in function.\n\nobject.__dict__\n\n A dictionary or other mapping object used to store an object\'s\n (writable) attributes.\n\nobject.__methods__\n\n Deprecated since version 2.2: Use the built-in function ``dir()``\n to get a list of an object\'s attributes. This attribute is no\n longer available.\n\nobject.__members__\n\n Deprecated since version 2.2: Use the built-in function ``dir()``\n to get a list of an object\'s attributes. This attribute is no\n longer available.\n\ninstance.__class__\n\n The class to which a class instance belongs.\n\nclass.__bases__\n\n The tuple of base classes of a class object.\n\nclass.__name__\n\n The name of the class or type.\n\nThe following attributes are only supported by *new-style class*es.\n\nclass.__mro__\n\n This attribute is a tuple of classes that are considered when\n looking for base classes during method resolution.\n\nclass.mro()\n\n This method can be overridden by a metaclass to customize the\n method resolution order for its instances. It is called at class\n instantiation, and its result is stored in ``__mro__``.\n\nclass.__subclasses__()\n\n Each new-style class keeps a list of weak references to its\n immediate subclasses. This method returns a list of all those\n references still alive. Example:\n\n >>> int.__subclasses__()\n []\n\n-[ Footnotes ]-\n\n[1] Additional information on these special methods may be found in\n the Python Reference Manual (*Basic customization*).\n\n[2] As a consequence, the list ``[1, 2]`` is considered equal to\n ``[1.0, 2.0]``, and similarly for tuples.\n\n[3] They must have since the parser can\'t tell the type of the\n operands.\n\n[4] Cased characters are those with general category property being\n one of "Lu" (Letter, uppercase), "Ll" (Letter, lowercase), or "Lt"\n (Letter, titlecase).\n\n[5] To format only a tuple you should therefore provide a singleton\n tuple whose only element is the tuple to be formatted.\n\n[6] The advantage of leaving the newline on is that returning an empty\n string is then an unambiguous EOF indication. It is also possible\n (in cases where it might matter, for example, if you want to make\n an exact copy of a file while scanning its lines) to tell whether\n the last line of a file ended in a newline or not (yes this\n happens!).\n', 'specialnames': '\nSpecial method names\n********************\n\nA class can implement certain operations that are invoked by special\nsyntax (such as arithmetic operations or subscripting and slicing) by\ndefining methods with special names. This is Python\'s approach to\n*operator overloading*, allowing classes to define their own behavior\nwith respect to language operators. For instance, if a class defines\na method named ``__getitem__()``, and ``x`` is an instance of this\nclass, then ``x[i]`` is roughly equivalent to ``x.__getitem__(i)`` for\nold-style classes and ``type(x).__getitem__(x, i)`` for new-style\nclasses. Except where mentioned, attempts to execute an operation\nraise an exception when no appropriate method is defined (typically\n``AttributeError`` or ``TypeError``).\n\nWhen implementing a class that emulates any built-in type, it is\nimportant that the emulation only be implemented to the degree that it\nmakes sense for the object being modelled. For example, some\nsequences may work well with retrieval of individual elements, but\nextracting a slice may not make sense. (One example of this is the\n``NodeList`` interface in the W3C\'s Document Object Model.)\n\n\nBasic customization\n===================\n\nobject.__new__(cls[, ...])\n\n Called to create a new instance of class *cls*. ``__new__()`` is a\n static method (special-cased so you need not declare it as such)\n that takes the class of which an instance was requested as its\n first argument. The remaining arguments are those passed to the\n object constructor expression (the call to the class). The return\n value of ``__new__()`` should be the new object instance (usually\n an instance of *cls*).\n\n Typical implementations create a new instance of the class by\n invoking the superclass\'s ``__new__()`` method using\n ``super(currentclass, cls).__new__(cls[, ...])`` with appropriate\n arguments and then modifying the newly-created instance as\n necessary before returning it.\n\n If ``__new__()`` returns an instance of *cls*, then the new\n instance\'s ``__init__()`` method will be invoked like\n ``__init__(self[, ...])``, where *self* is the new instance and the\n remaining arguments are the same as were passed to ``__new__()``.\n\n If ``__new__()`` does not return an instance of *cls*, then the new\n instance\'s ``__init__()`` method will not be invoked.\n\n ``__new__()`` is intended mainly to allow subclasses of immutable\n types (like int, str, or tuple) to customize instance creation. It\n is also commonly overridden in custom metaclasses in order to\n customize class creation.\n\nobject.__init__(self[, ...])\n\n Called when the instance is created. The arguments are those\n passed to the class constructor expression. If a base class has an\n ``__init__()`` method, the derived class\'s ``__init__()`` method,\n if any, must explicitly call it to ensure proper initialization of\n the base class part of the instance; for example:\n ``BaseClass.__init__(self, [args...])``. As a special constraint\n on constructors, no value may be returned; doing so will cause a\n ``TypeError`` to be raised at runtime.\n\nobject.__del__(self)\n\n Called when the instance is about to be destroyed. This is also\n called a destructor. If a base class has a ``__del__()`` method,\n the derived class\'s ``__del__()`` method, if any, must explicitly\n call it to ensure proper deletion of the base class part of the\n instance. Note that it is possible (though not recommended!) for\n the ``__del__()`` method to postpone destruction of the instance by\n creating a new reference to it. It may then be called at a later\n time when this new reference is deleted. It is not guaranteed that\n ``__del__()`` methods are called for objects that still exist when\n the interpreter exits.\n\n Note: ``del x`` doesn\'t directly call ``x.__del__()`` --- the former\n decrements the reference count for ``x`` by one, and the latter\n is only called when ``x``\'s reference count reaches zero. Some\n common situations that may prevent the reference count of an\n object from going to zero include: circular references between\n objects (e.g., a doubly-linked list or a tree data structure with\n parent and child pointers); a reference to the object on the\n stack frame of a function that caught an exception (the traceback\n stored in ``sys.exc_traceback`` keeps the stack frame alive); or\n a reference to the object on the stack frame that raised an\n unhandled exception in interactive mode (the traceback stored in\n ``sys.last_traceback`` keeps the stack frame alive). The first\n situation can only be remedied by explicitly breaking the cycles;\n the latter two situations can be resolved by storing ``None`` in\n ``sys.exc_traceback`` or ``sys.last_traceback``. Circular\n references which are garbage are detected when the option cycle\n detector is enabled (it\'s on by default), but can only be cleaned\n up if there are no Python-level ``__del__()`` methods involved.\n Refer to the documentation for the ``gc`` module for more\n information about how ``__del__()`` methods are handled by the\n cycle detector, particularly the description of the ``garbage``\n value.\n\n Warning: Due to the precarious circumstances under which ``__del__()``\n methods are invoked, exceptions that occur during their execution\n are ignored, and a warning is printed to ``sys.stderr`` instead.\n Also, when ``__del__()`` is invoked in response to a module being\n deleted (e.g., when execution of the program is done), other\n globals referenced by the ``__del__()`` method may already have\n been deleted or in the process of being torn down (e.g. the\n import machinery shutting down). For this reason, ``__del__()``\n methods should do the absolute minimum needed to maintain\n external invariants. Starting with version 1.5, Python\n guarantees that globals whose name begins with a single\n underscore are deleted from their module before other globals are\n deleted; if no other references to such globals exist, this may\n help in assuring that imported modules are still available at the\n time when the ``__del__()`` method is called.\n\n See also the *-R* command-line option.\n\nobject.__repr__(self)\n\n Called by the ``repr()`` built-in function and by string\n conversions (reverse quotes) to compute the "official" string\n representation of an object. If at all possible, this should look\n like a valid Python expression that could be used to recreate an\n object with the same value (given an appropriate environment). If\n this is not possible, a string of the form ``<...some useful\n description...>`` should be returned. The return value must be a\n string object. If a class defines ``__repr__()`` but not\n ``__str__()``, then ``__repr__()`` is also used when an "informal"\n string representation of instances of that class is required.\n\n This is typically used for debugging, so it is important that the\n representation is information-rich and unambiguous.\n\nobject.__str__(self)\n\n Called by the ``str()`` built-in function and by the ``print``\n statement to compute the "informal" string representation of an\n object. This differs from ``__repr__()`` in that it does not have\n to be a valid Python expression: a more convenient or concise\n representation may be used instead. The return value must be a\n string object.\n\nobject.__lt__(self, other)\nobject.__le__(self, other)\nobject.__eq__(self, other)\nobject.__ne__(self, other)\nobject.__gt__(self, other)\nobject.__ge__(self, other)\n\n New in version 2.1.\n\n These are the so-called "rich comparison" methods, and are called\n for comparison operators in preference to ``__cmp__()`` below. The\n correspondence between operator symbols and method names is as\n follows: ``xy`` call ``x.__ne__(y)``, ``x>y`` calls ``x.__gt__(y)``, and\n ``x>=y`` calls ``x.__ge__(y)``.\n\n A rich comparison method may return the singleton\n ``NotImplemented`` if it does not implement the operation for a\n given pair of arguments. By convention, ``False`` and ``True`` are\n returned for a successful comparison. However, these methods can\n return any value, so if the comparison operator is used in a\n Boolean context (e.g., in the condition of an ``if`` statement),\n Python will call ``bool()`` on the value to determine if the result\n is true or false.\n\n There are no implied relationships among the comparison operators.\n The truth of ``x==y`` does not imply that ``x!=y`` is false.\n Accordingly, when defining ``__eq__()``, one should also define\n ``__ne__()`` so that the operators will behave as expected. See\n the paragraph on ``__hash__()`` for some important notes on\n creating *hashable* objects which support custom comparison\n operations and are usable as dictionary keys.\n\n There are no swapped-argument versions of these methods (to be used\n when the left argument does not support the operation but the right\n argument does); rather, ``__lt__()`` and ``__gt__()`` are each\n other\'s reflection, ``__le__()`` and ``__ge__()`` are each other\'s\n reflection, and ``__eq__()`` and ``__ne__()`` are their own\n reflection.\n\n Arguments to rich comparison methods are never coerced.\n\n To automatically generate ordering operations from a single root\n operation, see ``functools.total_ordering()``.\n\nobject.__cmp__(self, other)\n\n Called by comparison operations if rich comparison (see above) is\n not defined. Should return a negative integer if ``self < other``,\n zero if ``self == other``, a positive integer if ``self > other``.\n If no ``__cmp__()``, ``__eq__()`` or ``__ne__()`` operation is\n defined, class instances are compared by object identity\n ("address"). See also the description of ``__hash__()`` for some\n important notes on creating *hashable* objects which support custom\n comparison operations and are usable as dictionary keys. (Note: the\n restriction that exceptions are not propagated by ``__cmp__()`` has\n been removed since Python 1.5.)\n\nobject.__rcmp__(self, other)\n\n Changed in version 2.1: No longer supported.\n\nobject.__hash__(self)\n\n Called by built-in function ``hash()`` and for operations on\n members of hashed collections including ``set``, ``frozenset``, and\n ``dict``. ``__hash__()`` should return an integer. The only\n required property is that objects which compare equal have the same\n hash value; it is advised to somehow mix together (e.g. using\n exclusive or) the hash values for the components of the object that\n also play a part in comparison of objects.\n\n If a class does not define a ``__cmp__()`` or ``__eq__()`` method\n it should not define a ``__hash__()`` operation either; if it\n defines ``__cmp__()`` or ``__eq__()`` but not ``__hash__()``, its\n instances will not be usable in hashed collections. If a class\n defines mutable objects and implements a ``__cmp__()`` or\n ``__eq__()`` method, it should not implement ``__hash__()``, since\n hashable collection implementations require that a object\'s hash\n value is immutable (if the object\'s hash value changes, it will be\n in the wrong hash bucket).\n\n User-defined classes have ``__cmp__()`` and ``__hash__()`` methods\n by default; with them, all objects compare unequal (except with\n themselves) and ``x.__hash__()`` returns ``id(x)``.\n\n Classes which inherit a ``__hash__()`` method from a parent class\n but change the meaning of ``__cmp__()`` or ``__eq__()`` such that\n the hash value returned is no longer appropriate (e.g. by switching\n to a value-based concept of equality instead of the default\n identity based equality) can explicitly flag themselves as being\n unhashable by setting ``__hash__ = None`` in the class definition.\n Doing so means that not only will instances of the class raise an\n appropriate ``TypeError`` when a program attempts to retrieve their\n hash value, but they will also be correctly identified as\n unhashable when checking ``isinstance(obj, collections.Hashable)``\n (unlike classes which define their own ``__hash__()`` to explicitly\n raise ``TypeError``).\n\n Changed in version 2.5: ``__hash__()`` may now also return a long\n integer object; the 32-bit integer is then derived from the hash of\n that object.\n\n Changed in version 2.6: ``__hash__`` may now be set to ``None`` to\n explicitly flag instances of a class as unhashable.\n\nobject.__nonzero__(self)\n\n Called to implement truth value testing and the built-in operation\n ``bool()``; should return ``False`` or ``True``, or their integer\n equivalents ``0`` or ``1``. When this method is not defined,\n ``__len__()`` is called, if it is defined, and the object is\n considered true if its result is nonzero. If a class defines\n neither ``__len__()`` nor ``__nonzero__()``, all its instances are\n considered true.\n\nobject.__unicode__(self)\n\n Called to implement ``unicode()`` built-in; should return a Unicode\n object. When this method is not defined, string conversion is\n attempted, and the result of string conversion is converted to\n Unicode using the system default encoding.\n\n\nCustomizing attribute access\n============================\n\nThe following methods can be defined to customize the meaning of\nattribute access (use of, assignment to, or deletion of ``x.name``)\nfor class instances.\n\nobject.__getattr__(self, name)\n\n Called when an attribute lookup has not found the attribute in the\n usual places (i.e. it is not an instance attribute nor is it found\n in the class tree for ``self``). ``name`` is the attribute name.\n This method should return the (computed) attribute value or raise\n an ``AttributeError`` exception.\n\n Note that if the attribute is found through the normal mechanism,\n ``__getattr__()`` is not called. (This is an intentional asymmetry\n between ``__getattr__()`` and ``__setattr__()``.) This is done both\n for efficiency reasons and because otherwise ``__getattr__()``\n would have no way to access other attributes of the instance. Note\n that at least for instance variables, you can fake total control by\n not inserting any values in the instance attribute dictionary (but\n instead inserting them in another object). See the\n ``__getattribute__()`` method below for a way to actually get total\n control in new-style classes.\n\nobject.__setattr__(self, name, value)\n\n Called when an attribute assignment is attempted. This is called\n instead of the normal mechanism (i.e. store the value in the\n instance dictionary). *name* is the attribute name, *value* is the\n value to be assigned to it.\n\n If ``__setattr__()`` wants to assign to an instance attribute, it\n should not simply execute ``self.name = value`` --- this would\n cause a recursive call to itself. Instead, it should insert the\n value in the dictionary of instance attributes, e.g.,\n ``self.__dict__[name] = value``. For new-style classes, rather\n than accessing the instance dictionary, it should call the base\n class method with the same name, for example,\n ``object.__setattr__(self, name, value)``.\n\nobject.__delattr__(self, name)\n\n Like ``__setattr__()`` but for attribute deletion instead of\n assignment. This should only be implemented if ``del obj.name`` is\n meaningful for the object.\n\n\nMore attribute access for new-style classes\n-------------------------------------------\n\nThe following methods only apply to new-style classes.\n\nobject.__getattribute__(self, name)\n\n Called unconditionally to implement attribute accesses for\n instances of the class. If the class also defines\n ``__getattr__()``, the latter will not be called unless\n ``__getattribute__()`` either calls it explicitly or raises an\n ``AttributeError``. This method should return the (computed)\n attribute value or raise an ``AttributeError`` exception. In order\n to avoid infinite recursion in this method, its implementation\n should always call the base class method with the same name to\n access any attributes it needs, for example,\n ``object.__getattribute__(self, name)``.\n\n Note: This method may still be bypassed when looking up special methods\n as the result of implicit invocation via language syntax or\n built-in functions. See *Special method lookup for new-style\n classes*.\n\n\nImplementing Descriptors\n------------------------\n\nThe following methods only apply when an instance of the class\ncontaining the method (a so-called *descriptor* class) appears in an\n*owner* class (the descriptor must be in either the owner\'s class\ndictionary or in the class dictionary for one of its parents). In the\nexamples below, "the attribute" refers to the attribute whose name is\nthe key of the property in the owner class\' ``__dict__``.\n\nobject.__get__(self, instance, owner)\n\n Called to get the attribute of the owner class (class attribute\n access) or of an instance of that class (instance attribute\n access). *owner* is always the owner class, while *instance* is the\n instance that the attribute was accessed through, or ``None`` when\n the attribute is accessed through the *owner*. This method should\n return the (computed) attribute value or raise an\n ``AttributeError`` exception.\n\nobject.__set__(self, instance, value)\n\n Called to set the attribute on an instance *instance* of the owner\n class to a new value, *value*.\n\nobject.__delete__(self, instance)\n\n Called to delete the attribute on an instance *instance* of the\n owner class.\n\n\nInvoking Descriptors\n--------------------\n\nIn general, a descriptor is an object attribute with "binding\nbehavior", one whose attribute access has been overridden by methods\nin the descriptor protocol: ``__get__()``, ``__set__()``, and\n``__delete__()``. If any of those methods are defined for an object,\nit is said to be a descriptor.\n\nThe default behavior for attribute access is to get, set, or delete\nthe attribute from an object\'s dictionary. For instance, ``a.x`` has a\nlookup chain starting with ``a.__dict__[\'x\']``, then\n``type(a).__dict__[\'x\']``, and continuing through the base classes of\n``type(a)`` excluding metaclasses.\n\nHowever, if the looked-up value is an object defining one of the\ndescriptor methods, then Python may override the default behavior and\ninvoke the descriptor method instead. Where this occurs in the\nprecedence chain depends on which descriptor methods were defined and\nhow they were called. Note that descriptors are only invoked for new\nstyle objects or classes (ones that subclass ``object()`` or\n``type()``).\n\nThe starting point for descriptor invocation is a binding, ``a.x``.\nHow the arguments are assembled depends on ``a``:\n\nDirect Call\n The simplest and least common call is when user code directly\n invokes a descriptor method: ``x.__get__(a)``.\n\nInstance Binding\n If binding to a new-style object instance, ``a.x`` is transformed\n into the call: ``type(a).__dict__[\'x\'].__get__(a, type(a))``.\n\nClass Binding\n If binding to a new-style class, ``A.x`` is transformed into the\n call: ``A.__dict__[\'x\'].__get__(None, A)``.\n\nSuper Binding\n If ``a`` is an instance of ``super``, then the binding ``super(B,\n obj).m()`` searches ``obj.__class__.__mro__`` for the base class\n ``A`` immediately preceding ``B`` and then invokes the descriptor\n with the call: ``A.__dict__[\'m\'].__get__(obj, obj.__class__)``.\n\nFor instance bindings, the precedence of descriptor invocation depends\non the which descriptor methods are defined. A descriptor can define\nany combination of ``__get__()``, ``__set__()`` and ``__delete__()``.\nIf it does not define ``__get__()``, then accessing the attribute will\nreturn the descriptor object itself unless there is a value in the\nobject\'s instance dictionary. If the descriptor defines ``__set__()``\nand/or ``__delete__()``, it is a data descriptor; if it defines\nneither, it is a non-data descriptor. Normally, data descriptors\ndefine both ``__get__()`` and ``__set__()``, while non-data\ndescriptors have just the ``__get__()`` method. Data descriptors with\n``__set__()`` and ``__get__()`` defined always override a redefinition\nin an instance dictionary. In contrast, non-data descriptors can be\noverridden by instances.\n\nPython methods (including ``staticmethod()`` and ``classmethod()``)\nare implemented as non-data descriptors. Accordingly, instances can\nredefine and override methods. This allows individual instances to\nacquire behaviors that differ from other instances of the same class.\n\nThe ``property()`` function is implemented as a data descriptor.\nAccordingly, instances cannot override the behavior of a property.\n\n\n__slots__\n---------\n\nBy default, instances of both old and new-style classes have a\ndictionary for attribute storage. This wastes space for objects\nhaving very few instance variables. The space consumption can become\nacute when creating large numbers of instances.\n\nThe default can be overridden by defining *__slots__* in a new-style\nclass definition. The *__slots__* declaration takes a sequence of\ninstance variables and reserves just enough space in each instance to\nhold a value for each variable. Space is saved because *__dict__* is\nnot created for each instance.\n\n__slots__\n\n This class variable can be assigned a string, iterable, or sequence\n of strings with variable names used by instances. If defined in a\n new-style class, *__slots__* reserves space for the declared\n variables and prevents the automatic creation of *__dict__* and\n *__weakref__* for each instance.\n\n New in version 2.2.\n\nNotes on using *__slots__*\n\n* When inheriting from a class without *__slots__*, the *__dict__*\n attribute of that class will always be accessible, so a *__slots__*\n definition in the subclass is meaningless.\n\n* Without a *__dict__* variable, instances cannot be assigned new\n variables not listed in the *__slots__* definition. Attempts to\n assign to an unlisted variable name raises ``AttributeError``. If\n dynamic assignment of new variables is desired, then add\n ``\'__dict__\'`` to the sequence of strings in the *__slots__*\n declaration.\n\n Changed in version 2.3: Previously, adding ``\'__dict__\'`` to the\n *__slots__* declaration would not enable the assignment of new\n attributes not specifically listed in the sequence of instance\n variable names.\n\n* Without a *__weakref__* variable for each instance, classes defining\n *__slots__* do not support weak references to its instances. If weak\n reference support is needed, then add ``\'__weakref__\'`` to the\n sequence of strings in the *__slots__* declaration.\n\n Changed in version 2.3: Previously, adding ``\'__weakref__\'`` to the\n *__slots__* declaration would not enable support for weak\n references.\n\n* *__slots__* are implemented at the class level by creating\n descriptors (*Implementing Descriptors*) for each variable name. As\n a result, class attributes cannot be used to set default values for\n instance variables defined by *__slots__*; otherwise, the class\n attribute would overwrite the descriptor assignment.\n\n* The action of a *__slots__* declaration is limited to the class\n where it is defined. As a result, subclasses will have a *__dict__*\n unless they also define *__slots__* (which must only contain names\n of any *additional* slots).\n\n* If a class defines a slot also defined in a base class, the instance\n variable defined by the base class slot is inaccessible (except by\n retrieving its descriptor directly from the base class). This\n renders the meaning of the program undefined. In the future, a\n check may be added to prevent this.\n\n* Nonempty *__slots__* does not work for classes derived from\n "variable-length" built-in types such as ``long``, ``str`` and\n ``tuple``.\n\n* Any non-string iterable may be assigned to *__slots__*. Mappings may\n also be used; however, in the future, special meaning may be\n assigned to the values corresponding to each key.\n\n* *__class__* assignment works only if both classes have the same\n *__slots__*.\n\n Changed in version 2.6: Previously, *__class__* assignment raised an\n error if either new or old class had *__slots__*.\n\n\nCustomizing class creation\n==========================\n\nBy default, new-style classes are constructed using ``type()``. A\nclass definition is read into a separate namespace and the value of\nclass name is bound to the result of ``type(name, bases, dict)``.\n\nWhen the class definition is read, if *__metaclass__* is defined then\nthe callable assigned to it will be called instead of ``type()``. This\nallows classes or functions to be written which monitor or alter the\nclass creation process:\n\n* Modifying the class dictionary prior to the class being created.\n\n* Returning an instance of another class -- essentially performing the\n role of a factory function.\n\nThese steps will have to be performed in the metaclass\'s ``__new__()``\nmethod -- ``type.__new__()`` can then be called from this method to\ncreate a class with different properties. This example adds a new\nelement to the class dictionary before creating the class:\n\n class metacls(type):\n def __new__(mcs, name, bases, dict):\n dict[\'foo\'] = \'metacls was here\'\n return type.__new__(mcs, name, bases, dict)\n\nYou can of course also override other class methods (or add new\nmethods); for example defining a custom ``__call__()`` method in the\nmetaclass allows custom behavior when the class is called, e.g. not\nalways creating a new instance.\n\n__metaclass__\n\n This variable can be any callable accepting arguments for ``name``,\n ``bases``, and ``dict``. Upon class creation, the callable is used\n instead of the built-in ``type()``.\n\n New in version 2.2.\n\nThe appropriate metaclass is determined by the following precedence\nrules:\n\n* If ``dict[\'__metaclass__\']`` exists, it is used.\n\n* Otherwise, if there is at least one base class, its metaclass is\n used (this looks for a *__class__* attribute first and if not found,\n uses its type).\n\n* Otherwise, if a global variable named __metaclass__ exists, it is\n used.\n\n* Otherwise, the old-style, classic metaclass (types.ClassType) is\n used.\n\nThe potential uses for metaclasses are boundless. Some ideas that have\nbeen explored including logging, interface checking, automatic\ndelegation, automatic property creation, proxies, frameworks, and\nautomatic resource locking/synchronization.\n\n\nCustomizing instance and subclass checks\n========================================\n\nNew in version 2.6.\n\nThe following methods are used to override the default behavior of the\n``isinstance()`` and ``issubclass()`` built-in functions.\n\nIn particular, the metaclass ``abc.ABCMeta`` implements these methods\nin order to allow the addition of Abstract Base Classes (ABCs) as\n"virtual base classes" to any class or type (including built-in\ntypes), including other ABCs.\n\nclass.__instancecheck__(self, instance)\n\n Return true if *instance* should be considered a (direct or\n indirect) instance of *class*. If defined, called to implement\n ``isinstance(instance, class)``.\n\nclass.__subclasscheck__(self, subclass)\n\n Return true if *subclass* should be considered a (direct or\n indirect) subclass of *class*. If defined, called to implement\n ``issubclass(subclass, class)``.\n\nNote that these methods are looked up on the type (metaclass) of a\nclass. They cannot be defined as class methods in the actual class.\nThis is consistent with the lookup of special methods that are called\non instances, only in this case the instance is itself a class.\n\nSee also:\n\n **PEP 3119** - Introducing Abstract Base Classes\n Includes the specification for customizing ``isinstance()`` and\n ``issubclass()`` behavior through ``__instancecheck__()`` and\n ``__subclasscheck__()``, with motivation for this functionality\n in the context of adding Abstract Base Classes (see the ``abc``\n module) to the language.\n\n\nEmulating callable objects\n==========================\n\nobject.__call__(self[, args...])\n\n Called when the instance is "called" as a function; if this method\n is defined, ``x(arg1, arg2, ...)`` is a shorthand for\n ``x.__call__(arg1, arg2, ...)``.\n\n\nEmulating container types\n=========================\n\nThe following methods can be defined to implement container objects.\nContainers usually are sequences (such as lists or tuples) or mappings\n(like dictionaries), but can represent other containers as well. The\nfirst set of methods is used either to emulate a sequence or to\nemulate a mapping; the difference is that for a sequence, the\nallowable keys should be the integers *k* for which ``0 <= k < N``\nwhere *N* is the length of the sequence, or slice objects, which\ndefine a range of items. (For backwards compatibility, the method\n``__getslice__()`` (see below) can also be defined to handle simple,\nbut not extended slices.) It is also recommended that mappings provide\nthe methods ``keys()``, ``values()``, ``items()``, ``has_key()``,\n``get()``, ``clear()``, ``setdefault()``, ``iterkeys()``,\n``itervalues()``, ``iteritems()``, ``pop()``, ``popitem()``,\n``copy()``, and ``update()`` behaving similar to those for Python\'s\nstandard dictionary objects. The ``UserDict`` module provides a\n``DictMixin`` class to help create those methods from a base set of\n``__getitem__()``, ``__setitem__()``, ``__delitem__()``, and\n``keys()``. Mutable sequences should provide methods ``append()``,\n``count()``, ``index()``, ``extend()``, ``insert()``, ``pop()``,\n``remove()``, ``reverse()`` and ``sort()``, like Python standard list\nobjects. Finally, sequence types should implement addition (meaning\nconcatenation) and multiplication (meaning repetition) by defining the\nmethods ``__add__()``, ``__radd__()``, ``__iadd__()``, ``__mul__()``,\n``__rmul__()`` and ``__imul__()`` described below; they should not\ndefine ``__coerce__()`` or other numerical operators. It is\nrecommended that both mappings and sequences implement the\n``__contains__()`` method to allow efficient use of the ``in``\noperator; for mappings, ``in`` should be equivalent of ``has_key()``;\nfor sequences, it should search through the values. It is further\nrecommended that both mappings and sequences implement the\n``__iter__()`` method to allow efficient iteration through the\ncontainer; for mappings, ``__iter__()`` should be the same as\n``iterkeys()``; for sequences, it should iterate through the values.\n\nobject.__len__(self)\n\n Called to implement the built-in function ``len()``. Should return\n the length of the object, an integer ``>=`` 0. Also, an object\n that doesn\'t define a ``__nonzero__()`` method and whose\n ``__len__()`` method returns zero is considered to be false in a\n Boolean context.\n\nobject.__getitem__(self, key)\n\n Called to implement evaluation of ``self[key]``. For sequence\n types, the accepted keys should be integers and slice objects.\n Note that the special interpretation of negative indexes (if the\n class wishes to emulate a sequence type) is up to the\n ``__getitem__()`` method. If *key* is of an inappropriate type,\n ``TypeError`` may be raised; if of a value outside the set of\n indexes for the sequence (after any special interpretation of\n negative values), ``IndexError`` should be raised. For mapping\n types, if *key* is missing (not in the container), ``KeyError``\n should be raised.\n\n Note: ``for`` loops expect that an ``IndexError`` will be raised for\n illegal indexes to allow proper detection of the end of the\n sequence.\n\nobject.__setitem__(self, key, value)\n\n Called to implement assignment to ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support changes to the values for keys, or if new keys\n can be added, or for sequences if elements can be replaced. The\n same exceptions should be raised for improper *key* values as for\n the ``__getitem__()`` method.\n\nobject.__delitem__(self, key)\n\n Called to implement deletion of ``self[key]``. Same note as for\n ``__getitem__()``. This should only be implemented for mappings if\n the objects support removal of keys, or for sequences if elements\n can be removed from the sequence. The same exceptions should be\n raised for improper *key* values as for the ``__getitem__()``\n method.\n\nobject.__iter__(self)\n\n This method is called when an iterator is required for a container.\n This method should return a new iterator object that can iterate\n over all the objects in the container. For mappings, it should\n iterate over the keys of the container, and should also be made\n available as the method ``iterkeys()``.\n\n Iterator objects also need to implement this method; they are\n required to return themselves. For more information on iterator\n objects, see *Iterator Types*.\n\nobject.__reversed__(self)\n\n Called (if present) by the ``reversed()`` built-in to implement\n reverse iteration. It should return a new iterator object that\n iterates over all the objects in the container in reverse order.\n\n If the ``__reversed__()`` method is not provided, the\n ``reversed()`` built-in will fall back to using the sequence\n protocol (``__len__()`` and ``__getitem__()``). Objects that\n support the sequence protocol should only provide\n ``__reversed__()`` if they can provide an implementation that is\n more efficient than the one provided by ``reversed()``.\n\n New in version 2.6.\n\nThe membership test operators (``in`` and ``not in``) are normally\nimplemented as an iteration through a sequence. However, container\nobjects can supply the following special method with a more efficient\nimplementation, which also does not require the object be a sequence.\n\nobject.__contains__(self, item)\n\n Called to implement membership test operators. Should return true\n if *item* is in *self*, false otherwise. For mapping objects, this\n should consider the keys of the mapping rather than the values or\n the key-item pairs.\n\n For objects that don\'t define ``__contains__()``, the membership\n test first tries iteration via ``__iter__()``, then the old\n sequence iteration protocol via ``__getitem__()``, see *this\n section in the language reference*.\n\n\nAdditional methods for emulation of sequence types\n==================================================\n\nThe following optional methods can be defined to further emulate\nsequence objects. Immutable sequences methods should at most only\ndefine ``__getslice__()``; mutable sequences might define all three\nmethods.\n\nobject.__getslice__(self, i, j)\n\n Deprecated since version 2.0: Support slice objects as parameters\n to the ``__getitem__()`` method. (However, built-in types in\n CPython currently still implement ``__getslice__()``. Therefore,\n you have to override it in derived classes when implementing\n slicing.)\n\n Called to implement evaluation of ``self[i:j]``. The returned\n object should be of the same type as *self*. Note that missing *i*\n or *j* in the slice expression are replaced by zero or\n ``sys.maxint``, respectively. If negative indexes are used in the\n slice, the length of the sequence is added to that index. If the\n instance does not implement the ``__len__()`` method, an\n ``AttributeError`` is raised. No guarantee is made that indexes\n adjusted this way are not still negative. Indexes which are\n greater than the length of the sequence are not modified. If no\n ``__getslice__()`` is found, a slice object is created instead, and\n passed to ``__getitem__()`` instead.\n\nobject.__setslice__(self, i, j, sequence)\n\n Called to implement assignment to ``self[i:j]``. Same notes for *i*\n and *j* as for ``__getslice__()``.\n\n This method is deprecated. If no ``__setslice__()`` is found, or\n for extended slicing of the form ``self[i:j:k]``, a slice object is\n created, and passed to ``__setitem__()``, instead of\n ``__setslice__()`` being called.\n\nobject.__delslice__(self, i, j)\n\n Called to implement deletion of ``self[i:j]``. Same notes for *i*\n and *j* as for ``__getslice__()``. This method is deprecated. If no\n ``__delslice__()`` is found, or for extended slicing of the form\n ``self[i:j:k]``, a slice object is created, and passed to\n ``__delitem__()``, instead of ``__delslice__()`` being called.\n\nNotice that these methods are only invoked when a single slice with a\nsingle colon is used, and the slice method is available. For slice\noperations involving extended slice notation, or in absence of the\nslice methods, ``__getitem__()``, ``__setitem__()`` or\n``__delitem__()`` is called with a slice object as argument.\n\nThe following example demonstrate how to make your program or module\ncompatible with earlier versions of Python (assuming that methods\n``__getitem__()``, ``__setitem__()`` and ``__delitem__()`` support\nslice objects as arguments):\n\n class MyClass:\n ...\n def __getitem__(self, index):\n ...\n def __setitem__(self, index, value):\n ...\n def __delitem__(self, index):\n ...\n\n if sys.version_info < (2, 0):\n # They won\'t be defined if version is at least 2.0 final\n\n def __getslice__(self, i, j):\n return self[max(0, i):max(0, j):]\n def __setslice__(self, i, j, seq):\n self[max(0, i):max(0, j):] = seq\n def __delslice__(self, i, j):\n del self[max(0, i):max(0, j):]\n ...\n\nNote the calls to ``max()``; these are necessary because of the\nhandling of negative indices before the ``__*slice__()`` methods are\ncalled. When negative indexes are used, the ``__*item__()`` methods\nreceive them as provided, but the ``__*slice__()`` methods get a\n"cooked" form of the index values. For each negative index value, the\nlength of the sequence is added to the index before calling the method\n(which may still result in a negative index); this is the customary\nhandling of negative indexes by the built-in sequence types, and the\n``__*item__()`` methods are expected to do this as well. However,\nsince they should already be doing that, negative indexes cannot be\npassed in; they must be constrained to the bounds of the sequence\nbefore being passed to the ``__*item__()`` methods. Calling ``max(0,\ni)`` conveniently returns the proper value.\n\n\nEmulating numeric types\n=======================\n\nThe following methods can be defined to emulate numeric objects.\nMethods corresponding to operations that are not supported by the\nparticular kind of number implemented (e.g., bitwise operations for\nnon-integral numbers) should be left undefined.\n\nobject.__add__(self, other)\nobject.__sub__(self, other)\nobject.__mul__(self, other)\nobject.__floordiv__(self, other)\nobject.__mod__(self, other)\nobject.__divmod__(self, other)\nobject.__pow__(self, other[, modulo])\nobject.__lshift__(self, other)\nobject.__rshift__(self, other)\nobject.__and__(self, other)\nobject.__xor__(self, other)\nobject.__or__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``//``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For\n instance, to evaluate the expression ``x + y``, where *x* is an\n instance of a class that has an ``__add__()`` method,\n ``x.__add__(y)`` is called. The ``__divmod__()`` method should be\n the equivalent to using ``__floordiv__()`` and ``__mod__()``; it\n should not be related to ``__truediv__()`` (described below). Note\n that ``__pow__()`` should be defined to accept an optional third\n argument if the ternary version of the built-in ``pow()`` function\n is to be supported.\n\n If one of those methods does not support the operation with the\n supplied arguments, it should return ``NotImplemented``.\n\nobject.__div__(self, other)\nobject.__truediv__(self, other)\n\n The division operator (``/``) is implemented by these methods. The\n ``__truediv__()`` method is used when ``__future__.division`` is in\n effect, otherwise ``__div__()`` is used. If only one of these two\n methods is defined, the object will not support division in the\n alternate context; ``TypeError`` will be raised instead.\n\nobject.__radd__(self, other)\nobject.__rsub__(self, other)\nobject.__rmul__(self, other)\nobject.__rdiv__(self, other)\nobject.__rtruediv__(self, other)\nobject.__rfloordiv__(self, other)\nobject.__rmod__(self, other)\nobject.__rdivmod__(self, other)\nobject.__rpow__(self, other)\nobject.__rlshift__(self, other)\nobject.__rrshift__(self, other)\nobject.__rand__(self, other)\nobject.__rxor__(self, other)\nobject.__ror__(self, other)\n\n These methods are called to implement the binary arithmetic\n operations (``+``, ``-``, ``*``, ``/``, ``%``, ``divmod()``,\n ``pow()``, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with\n reflected (swapped) operands. These functions are only called if\n the left operand does not support the corresponding operation and\n the operands are of different types. [2] For instance, to evaluate\n the expression ``x - y``, where *y* is an instance of a class that\n has an ``__rsub__()`` method, ``y.__rsub__(x)`` is called if\n ``x.__sub__(y)`` returns *NotImplemented*.\n\n Note that ternary ``pow()`` will not try calling ``__rpow__()``\n (the coercion rules would become too complicated).\n\n Note: If the right operand\'s type is a subclass of the left operand\'s\n type and that subclass provides the reflected method for the\n operation, this method will be called before the left operand\'s\n non-reflected method. This behavior allows subclasses to\n override their ancestors\' operations.\n\nobject.__iadd__(self, other)\nobject.__isub__(self, other)\nobject.__imul__(self, other)\nobject.__idiv__(self, other)\nobject.__itruediv__(self, other)\nobject.__ifloordiv__(self, other)\nobject.__imod__(self, other)\nobject.__ipow__(self, other[, modulo])\nobject.__ilshift__(self, other)\nobject.__irshift__(self, other)\nobject.__iand__(self, other)\nobject.__ixor__(self, other)\nobject.__ior__(self, other)\n\n These methods are called to implement the augmented arithmetic\n assignments (``+=``, ``-=``, ``*=``, ``/=``, ``//=``, ``%=``,\n ``**=``, ``<<=``, ``>>=``, ``&=``, ``^=``, ``|=``). These methods\n should attempt to do the operation in-place (modifying *self*) and\n return the result (which could be, but does not have to be,\n *self*). If a specific method is not defined, the augmented\n assignment falls back to the normal methods. For instance, to\n execute the statement ``x += y``, where *x* is an instance of a\n class that has an ``__iadd__()`` method, ``x.__iadd__(y)`` is\n called. If *x* is an instance of a class that does not define a\n ``__iadd__()`` method, ``x.__add__(y)`` and ``y.__radd__(x)`` are\n considered, as with the evaluation of ``x + y``.\n\nobject.__neg__(self)\nobject.__pos__(self)\nobject.__abs__(self)\nobject.__invert__(self)\n\n Called to implement the unary arithmetic operations (``-``, ``+``,\n ``abs()`` and ``~``).\n\nobject.__complex__(self)\nobject.__int__(self)\nobject.__long__(self)\nobject.__float__(self)\n\n Called to implement the built-in functions ``complex()``,\n ``int()``, ``long()``, and ``float()``. Should return a value of\n the appropriate type.\n\nobject.__oct__(self)\nobject.__hex__(self)\n\n Called to implement the built-in functions ``oct()`` and ``hex()``.\n Should return a string value.\n\nobject.__index__(self)\n\n Called to implement ``operator.index()``. Also called whenever\n Python needs an integer object (such as in slicing). Must return\n an integer (int or long).\n\n New in version 2.5.\n\nobject.__coerce__(self, other)\n\n Called to implement "mixed-mode" numeric arithmetic. Should either\n return a 2-tuple containing *self* and *other* converted to a\n common numeric type, or ``None`` if conversion is impossible. When\n the common type would be the type of ``other``, it is sufficient to\n return ``None``, since the interpreter will also ask the other\n object to attempt a coercion (but sometimes, if the implementation\n of the other type cannot be changed, it is useful to do the\n conversion to the other type here). A return value of\n ``NotImplemented`` is equivalent to returning ``None``.\n\n\nCoercion rules\n==============\n\nThis section used to document the rules for coercion. As the language\nhas evolved, the coercion rules have become hard to document\nprecisely; documenting what one version of one particular\nimplementation does is undesirable. Instead, here are some informal\nguidelines regarding coercion. In Python 3, coercion will not be\nsupported.\n\n* If the left operand of a % operator is a string or Unicode object,\n no coercion takes place and the string formatting operation is\n invoked instead.\n\n* It is no longer recommended to define a coercion operation. Mixed-\n mode operations on types that don\'t define coercion pass the\n original arguments to the operation.\n\n* New-style classes (those derived from ``object``) never invoke the\n ``__coerce__()`` method in response to a binary operator; the only\n time ``__coerce__()`` is invoked is when the built-in function\n ``coerce()`` is called.\n\n* For most intents and purposes, an operator that returns\n ``NotImplemented`` is treated the same as one that is not\n implemented at all.\n\n* Below, ``__op__()`` and ``__rop__()`` are used to signify the\n generic method names corresponding to an operator; ``__iop__()`` is\n used for the corresponding in-place operator. For example, for the\n operator \'``+``\', ``__add__()`` and ``__radd__()`` are used for the\n left and right variant of the binary operator, and ``__iadd__()``\n for the in-place variant.\n\n* For objects *x* and *y*, first ``x.__op__(y)`` is tried. If this is\n not implemented or returns ``NotImplemented``, ``y.__rop__(x)`` is\n tried. If this is also not implemented or returns\n ``NotImplemented``, a ``TypeError`` exception is raised. But see\n the following exception:\n\n* Exception to the previous item: if the left operand is an instance\n of a built-in type or a new-style class, and the right operand is an\n instance of a proper subclass of that type or class and overrides\n the base\'s ``__rop__()`` method, the right operand\'s ``__rop__()``\n method is tried *before* the left operand\'s ``__op__()`` method.\n\n This is done so that a subclass can completely override binary\n operators. Otherwise, the left operand\'s ``__op__()`` method would\n always accept the right operand: when an instance of a given class\n is expected, an instance of a subclass of that class is always\n acceptable.\n\n* When either operand type defines a coercion, this coercion is called\n before that type\'s ``__op__()`` or ``__rop__()`` method is called,\n but no sooner. If the coercion returns an object of a different\n type for the operand whose coercion is invoked, part of the process\n is redone using the new object.\n\n* When an in-place operator (like \'``+=``\') is used, if the left\n operand implements ``__iop__()``, it is invoked without any\n coercion. When the operation falls back to ``__op__()`` and/or\n ``__rop__()``, the normal coercion rules apply.\n\n* In ``x + y``, if *x* is a sequence that implements sequence\n concatenation, sequence concatenation is invoked.\n\n* In ``x * y``, if one operand is a sequence that implements sequence\n repetition, and the other is an integer (``int`` or ``long``),\n sequence repetition is invoked.\n\n* Rich comparisons (implemented by methods ``__eq__()`` and so on)\n never use coercion. Three-way comparison (implemented by\n ``__cmp__()``) does use coercion under the same conditions as other\n binary operations use it.\n\n* In the current implementation, the built-in numeric types ``int``,\n ``long``, ``float``, and ``complex`` do not use coercion. All these\n types implement a ``__coerce__()`` method, for use by the built-in\n ``coerce()`` function.\n\n Changed in version 2.7.\n\n\nWith Statement Context Managers\n===============================\n\nNew in version 2.5.\n\nA *context manager* is an object that defines the runtime context to\nbe established when executing a ``with`` statement. The context\nmanager handles the entry into, and the exit from, the desired runtime\ncontext for the execution of the block of code. Context managers are\nnormally invoked using the ``with`` statement (described in section\n*The with statement*), but can also be used by directly invoking their\nmethods.\n\nTypical uses of context managers include saving and restoring various\nkinds of global state, locking and unlocking resources, closing opened\nfiles, etc.\n\nFor more information on context managers, see *Context Manager Types*.\n\nobject.__enter__(self)\n\n Enter the runtime context related to this object. The ``with``\n statement will bind this method\'s return value to the target(s)\n specified in the ``as`` clause of the statement, if any.\n\nobject.__exit__(self, exc_type, exc_value, traceback)\n\n Exit the runtime context related to this object. The parameters\n describe the exception that caused the context to be exited. If the\n context was exited without an exception, all three arguments will\n be ``None``.\n\n If an exception is supplied, and the method wishes to suppress the\n exception (i.e., prevent it from being propagated), it should\n return a true value. Otherwise, the exception will be processed\n normally upon exit from this method.\n\n Note that ``__exit__()`` methods should not reraise the passed-in\n exception; this is the caller\'s responsibility.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n\n\nSpecial method lookup for old-style classes\n===========================================\n\nFor old-style classes, special methods are always looked up in exactly\nthe same way as any other method or attribute. This is the case\nregardless of whether the method is being looked up explicitly as in\n``x.__getitem__(i)`` or implicitly as in ``x[i]``.\n\nThis behaviour means that special methods may exhibit different\nbehaviour for different instances of a single old-style class if the\nappropriate special attributes are set differently:\n\n >>> class C:\n ... pass\n ...\n >>> c1 = C()\n >>> c2 = C()\n >>> c1.__len__ = lambda: 5\n >>> c2.__len__ = lambda: 9\n >>> len(c1)\n 5\n >>> len(c2)\n 9\n\n\nSpecial method lookup for new-style classes\n===========================================\n\nFor new-style classes, implicit invocations of special methods are\nonly guaranteed to work correctly if defined on an object\'s type, not\nin the object\'s instance dictionary. That behaviour is the reason why\nthe following code raises an exception (unlike the equivalent example\nwith old-style classes):\n\n >>> class C(object):\n ... pass\n ...\n >>> c = C()\n >>> c.__len__ = lambda: 5\n >>> len(c)\n Traceback (most recent call last):\n File "", line 1, in \n TypeError: object of type \'C\' has no len()\n\nThe rationale behind this behaviour lies with a number of special\nmethods such as ``__hash__()`` and ``__repr__()`` that are implemented\nby all objects, including type objects. If the implicit lookup of\nthese methods used the conventional lookup process, they would fail\nwhen invoked on the type object itself:\n\n >>> 1 .__hash__() == hash(1)\n True\n >>> int.__hash__() == hash(int)\n Traceback (most recent call last):\n File "", line 1, in \n TypeError: descriptor \'__hash__\' of \'int\' object needs an argument\n\nIncorrectly attempting to invoke an unbound method of a class in this\nway is sometimes referred to as \'metaclass confusion\', and is avoided\nby bypassing the instance when looking up special methods:\n\n >>> type(1).__hash__(1) == hash(1)\n True\n >>> type(int).__hash__(int) == hash(int)\n True\n\nIn addition to bypassing any instance attributes in the interest of\ncorrectness, implicit special method lookup generally also bypasses\nthe ``__getattribute__()`` method even of the object\'s metaclass:\n\n >>> class Meta(type):\n ... def __getattribute__(*args):\n ... print "Metaclass getattribute invoked"\n ... return type.__getattribute__(*args)\n ...\n >>> class C(object):\n ... __metaclass__ = Meta\n ... def __len__(self):\n ... return 10\n ... def __getattribute__(*args):\n ... print "Class getattribute invoked"\n ... return object.__getattribute__(*args)\n ...\n >>> c = C()\n >>> c.__len__() # Explicit lookup via instance\n Class getattribute invoked\n 10\n >>> type(c).__len__(c) # Explicit lookup via type\n Metaclass getattribute invoked\n 10\n >>> len(c) # Implicit lookup\n 10\n\nBypassing the ``__getattribute__()`` machinery in this fashion\nprovides significant scope for speed optimisations within the\ninterpreter, at the cost of some flexibility in the handling of\nspecial methods (the special method *must* be set on the class object\nitself in order to be consistently invoked by the interpreter).\n\n-[ Footnotes ]-\n\n[1] It *is* possible in some cases to change an object\'s type, under\n certain controlled conditions. It generally isn\'t a good idea\n though, since it can lead to some very strange behaviour if it is\n handled incorrectly.\n\n[2] For operands of the same type, it is assumed that if the non-\n reflected method (such as ``__add__()``) fails the operation is\n not supported, which is why the reflected method is not called.\n', - 'string-methods': '\nString Methods\n**************\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support. Some of them are also available on\n``bytearray`` objects.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbytearray, buffer, xrange* section. To output formatted strings use\ntemplate strings or the ``%`` operator described in the *String\nFormatting Operations* section. Also, see the ``re`` module for string\nfunctions based on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters [4] in the string are lowercase\n and there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters [4] in the string are uppercase\n and there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string with all the cased characters [4]\n converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified or ``-1``, then there is\n no limit on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. This method uses the *universal newlines* approach to\n splitting lines. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\n For example, ``\'ab c\\n\\nde fg\\rkl\\r\\n\'.splitlines()`` returns\n ``[\'ab c\', \'\', \'de fg\', \'kl\']``, while the same call with\n ``splitlines(True)`` returns ``[\'ab c\\n\', \'\\n\', \'de fg\\r\',\n \'kl\\r\\n\']``.\n\n Unlike ``split()`` when a delimiter string *sep* is given, this\n method returns an empty list for the empty string, and a terminal\n line break does not result in an extra line.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n ... return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n ... lambda mo: mo.group(0)[0].upper() +\n ... mo.group(0)[1:].lower(),\n ... s)\n ...\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string with all the cased characters [4]\n converted to uppercase. Note that ``str.upper().isupper()`` might\n be ``False`` if ``s`` contains uncased characters or if the Unicode\n category of the resulting character(s) is not "Lu" (Letter,\n uppercase), but e.g. "Lt" (Letter, titlecase).\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than or equal to ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that can be used to form decimal-radix numbers,\n e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n', + 'string-methods': '\nString Methods\n**************\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support. Some of them are also available on\n``bytearray`` objects.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbytearray, buffer, xrange* section. To output formatted strings use\ntemplate strings or the ``%`` operator described in the *String\nFormatting Operations* section. Also, see the ``re`` module for string\nfunctions based on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. Tab positions occur every *tabsize* characters\n (default is 8, giving tab positions at columns 0, 8, 16 and so on).\n To expand the string, the current column is set to zero and the\n string is examined character by character. If the character is a\n tab (``\\t``), one or more space characters are inserted in the\n result until the current column is equal to the next tab position.\n (The tab character itself is not copied.) If the character is a\n newline (``\\n``) or return (``\\r``), it is copied and the current\n column is reset to zero. Any other character is copied unchanged\n and the current column is incremented by one regardless of how the\n character is represented when printed.\n\n >>> \'01\\t012\\t0123\\t01234\'.expandtabs()\n \'01 012 0123 01234\'\n >>> \'01\\t012\\t0123\\t01234\'.expandtabs(4)\n \'01 012 0123 01234\'\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters [4] in the string are lowercase\n and there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters [4] in the string are uppercase\n and there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string with all the cased characters [4]\n converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified or ``-1``, then there is\n no limit on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. This method uses the *universal newlines* approach to\n splitting lines. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\n For example, ``\'ab c\\n\\nde fg\\rkl\\r\\n\'.splitlines()`` returns\n ``[\'ab c\', \'\', \'de fg\', \'kl\']``, while the same call with\n ``splitlines(True)`` returns ``[\'ab c\\n\', \'\\n\', \'de fg\\r\',\n \'kl\\r\\n\']``.\n\n Unlike ``split()`` when a delimiter string *sep* is given, this\n method returns an empty list for the empty string, and a terminal\n line break does not result in an extra line.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n ... return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n ... lambda mo: mo.group(0)[0].upper() +\n ... mo.group(0)[1:].lower(),\n ... s)\n ...\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string with all the cased characters [4]\n converted to uppercase. Note that ``str.upper().isupper()`` might\n be ``False`` if ``s`` contains uncased characters or if the Unicode\n category of the resulting character(s) is not "Lu" (Letter,\n uppercase), but e.g. "Lt" (Letter, titlecase).\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than or equal to ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that can be used to form decimal-radix numbers,\n e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n', 'strings': '\nString literals\n***************\n\nString literals are described by the following lexical definitions:\n\n stringliteral ::= [stringprefix](shortstring | longstring)\n stringprefix ::= "r" | "u" | "ur" | "R" | "U" | "UR" | "Ur" | "uR"\n | "b" | "B" | "br" | "Br" | "bR" | "BR"\n shortstring ::= "\'" shortstringitem* "\'" | \'"\' shortstringitem* \'"\'\n longstring ::= "\'\'\'" longstringitem* "\'\'\'"\n | \'"""\' longstringitem* \'"""\'\n shortstringitem ::= shortstringchar | escapeseq\n longstringitem ::= longstringchar | escapeseq\n shortstringchar ::= \n longstringchar ::= \n escapeseq ::= "\\" \n\nOne syntactic restriction not indicated by these productions is that\nwhitespace is not allowed between the ``stringprefix`` and the rest of\nthe string literal. The source character set is defined by the\nencoding declaration; it is ASCII if no encoding declaration is given\nin the source file; see section *Encoding declarations*.\n\nIn plain English: String literals can be enclosed in matching single\nquotes (``\'``) or double quotes (``"``). They can also be enclosed in\nmatching groups of three single or double quotes (these are generally\nreferred to as *triple-quoted strings*). The backslash (``\\``)\ncharacter is used to escape characters that otherwise have a special\nmeaning, such as newline, backslash itself, or the quote character.\nString literals may optionally be prefixed with a letter ``\'r\'`` or\n``\'R\'``; such strings are called *raw strings* and use different rules\nfor interpreting backslash escape sequences. A prefix of ``\'u\'`` or\n``\'U\'`` makes the string a Unicode string. Unicode strings use the\nUnicode character set as defined by the Unicode Consortium and ISO\n10646. Some additional escape sequences, described below, are\navailable in Unicode strings. A prefix of ``\'b\'`` or ``\'B\'`` is\nignored in Python 2; it indicates that the literal should become a\nbytes literal in Python 3 (e.g. when code is automatically converted\nwith 2to3). A ``\'u\'`` or ``\'b\'`` prefix may be followed by an ``\'r\'``\nprefix.\n\nIn triple-quoted strings, unescaped newlines and quotes are allowed\n(and are retained), except that three unescaped quotes in a row\nterminate the string. (A "quote" is the character used to open the\nstring, i.e. either ``\'`` or ``"``.)\n\nUnless an ``\'r\'`` or ``\'R\'`` prefix is present, escape sequences in\nstrings are interpreted according to rules similar to those used by\nStandard C. The recognized escape sequences are:\n\n+-------------------+-----------------------------------+---------+\n| Escape Sequence | Meaning | Notes |\n+===================+===================================+=========+\n| ``\\newline`` | Ignored | |\n+-------------------+-----------------------------------+---------+\n| ``\\\\`` | Backslash (``\\``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\\'`` | Single quote (``\'``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\"`` | Double quote (``"``) | |\n+-------------------+-----------------------------------+---------+\n| ``\\a`` | ASCII Bell (BEL) | |\n+-------------------+-----------------------------------+---------+\n| ``\\b`` | ASCII Backspace (BS) | |\n+-------------------+-----------------------------------+---------+\n| ``\\f`` | ASCII Formfeed (FF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\n`` | ASCII Linefeed (LF) | |\n+-------------------+-----------------------------------+---------+\n| ``\\N{name}`` | Character named *name* in the | |\n| | Unicode database (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\r`` | ASCII Carriage Return (CR) | |\n+-------------------+-----------------------------------+---------+\n| ``\\t`` | ASCII Horizontal Tab (TAB) | |\n+-------------------+-----------------------------------+---------+\n| ``\\uxxxx`` | Character with 16-bit hex value | (1) |\n| | *xxxx* (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\Uxxxxxxxx`` | Character with 32-bit hex value | (2) |\n| | *xxxxxxxx* (Unicode only) | |\n+-------------------+-----------------------------------+---------+\n| ``\\v`` | ASCII Vertical Tab (VT) | |\n+-------------------+-----------------------------------+---------+\n| ``\\ooo`` | Character with octal value *ooo* | (3,5) |\n+-------------------+-----------------------------------+---------+\n| ``\\xhh`` | Character with hex value *hh* | (4,5) |\n+-------------------+-----------------------------------+---------+\n\nNotes:\n\n1. Individual code units which form parts of a surrogate pair can be\n encoded using this escape sequence.\n\n2. Any Unicode character can be encoded this way, but characters\n outside the Basic Multilingual Plane (BMP) will be encoded using a\n surrogate pair if Python is compiled to use 16-bit code units (the\n default). Individual code units which form parts of a surrogate\n pair can be encoded using this escape sequence.\n\n3. As in Standard C, up to three octal digits are accepted.\n\n4. Unlike in Standard C, exactly two hex digits are required.\n\n5. In a string literal, hexadecimal and octal escapes denote the byte\n with the given value; it is not necessary that the byte encodes a\n character in the source character set. In a Unicode literal, these\n escapes denote a Unicode character with the given value.\n\nUnlike Standard C, all unrecognized escape sequences are left in the\nstring unchanged, i.e., *the backslash is left in the string*. (This\nbehavior is useful when debugging: if an escape sequence is mistyped,\nthe resulting output is more easily recognized as broken.) It is also\nimportant to note that the escape sequences marked as "(Unicode only)"\nin the table above fall into the category of unrecognized escapes for\nnon-Unicode string literals.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is present, a character following a\nbackslash is included in the string without change, and *all\nbackslashes are left in the string*. For example, the string literal\n``r"\\n"`` consists of two characters: a backslash and a lowercase\n``\'n\'``. String quotes can be escaped with a backslash, but the\nbackslash remains in the string; for example, ``r"\\""`` is a valid\nstring literal consisting of two characters: a backslash and a double\nquote; ``r"\\"`` is not a valid string literal (even a raw string\ncannot end in an odd number of backslashes). Specifically, *a raw\nstring cannot end in a single backslash* (since the backslash would\nescape the following quote character). Note also that a single\nbackslash followed by a newline is interpreted as those two characters\nas part of the string, *not* as a line continuation.\n\nWhen an ``\'r\'`` or ``\'R\'`` prefix is used in conjunction with a\n``\'u\'`` or ``\'U\'`` prefix, then the ``\\uXXXX`` and ``\\UXXXXXXXX``\nescape sequences are processed while *all other backslashes are left\nin the string*. For example, the string literal ``ur"\\u0062\\n"``\nconsists of three Unicode characters: \'LATIN SMALL LETTER B\', \'REVERSE\nSOLIDUS\', and \'LATIN SMALL LETTER N\'. Backslashes can be escaped with\na preceding backslash; however, both remain in the string. As a\nresult, ``\\uXXXX`` escape sequences are only recognized when there are\nan odd number of backslashes.\n', 'subscriptions': '\nSubscriptions\n*************\n\nA subscription selects an item of a sequence (string, tuple or list)\nor mapping (dictionary) object:\n\n subscription ::= primary "[" expression_list "]"\n\nThe primary must evaluate to an object of a sequence or mapping type.\n\nIf the primary is a mapping, the expression list must evaluate to an\nobject whose value is one of the keys of the mapping, and the\nsubscription selects the value in the mapping that corresponds to that\nkey. (The expression list is a tuple except if it has exactly one\nitem.)\n\nIf the primary is a sequence, the expression (list) must evaluate to a\nplain integer. If this value is negative, the length of the sequence\nis added to it (so that, e.g., ``x[-1]`` selects the last item of\n``x``.) The resulting value must be a nonnegative integer less than\nthe number of items in the sequence, and the subscription selects the\nitem whose index is that value (counting from zero).\n\nA string\'s items are characters. A character is not a separate data\ntype but a string of exactly one character.\n', 'truth': "\nTruth Value Testing\n*******************\n\nAny object can be tested for truth value, for use in an ``if`` or\n``while`` condition or as operand of the Boolean operations below. The\nfollowing values are considered false:\n\n* ``None``\n\n* ``False``\n\n* zero of any numeric type, for example, ``0``, ``0L``, ``0.0``,\n ``0j``.\n\n* any empty sequence, for example, ``''``, ``()``, ``[]``.\n\n* any empty mapping, for example, ``{}``.\n\n* instances of user-defined classes, if the class defines a\n ``__nonzero__()`` or ``__len__()`` method, when that method returns\n the integer zero or ``bool`` value ``False``. [1]\n\nAll other values are considered true --- so objects of many types are\nalways true.\n\nOperations and built-in functions that have a Boolean result always\nreturn ``0`` or ``False`` for false and ``1`` or ``True`` for true,\nunless otherwise stated. (Important exception: the Boolean operations\n``or`` and ``and`` always return one of their operands.)\n", 'try': '\nThe ``try`` statement\n*********************\n\nThe ``try`` statement specifies exception handlers and/or cleanup code\nfor a group of statements:\n\n try_stmt ::= try1_stmt | try2_stmt\n try1_stmt ::= "try" ":" suite\n ("except" [expression [("as" | ",") target]] ":" suite)+\n ["else" ":" suite]\n ["finally" ":" suite]\n try2_stmt ::= "try" ":" suite\n "finally" ":" suite\n\nChanged in version 2.5: In previous versions of Python,\n``try``...``except``...``finally`` did not work. ``try``...``except``\nhad to be nested in ``try``...``finally``.\n\nThe ``except`` clause(s) specify one or more exception handlers. When\nno exception occurs in the ``try`` clause, no exception handler is\nexecuted. When an exception occurs in the ``try`` suite, a search for\nan exception handler is started. This search inspects the except\nclauses in turn until one is found that matches the exception. An\nexpression-less except clause, if present, must be last; it matches\nany exception. For an except clause with an expression, that\nexpression is evaluated, and the clause matches the exception if the\nresulting object is "compatible" with the exception. An object is\ncompatible with an exception if it is the class or a base class of the\nexception object, or a tuple containing an item compatible with the\nexception.\n\nIf no except clause matches the exception, the search for an exception\nhandler continues in the surrounding code and on the invocation stack.\n[1]\n\nIf the evaluation of an expression in the header of an except clause\nraises an exception, the original search for a handler is canceled and\na search starts for the new exception in the surrounding code and on\nthe call stack (it is treated as if the entire ``try`` statement\nraised the exception).\n\nWhen a matching except clause is found, the exception is assigned to\nthe target specified in that except clause, if present, and the except\nclause\'s suite is executed. All except clauses must have an\nexecutable block. When the end of this block is reached, execution\ncontinues normally after the entire try statement. (This means that\nif two nested handlers exist for the same exception, and the exception\noccurs in the try clause of the inner handler, the outer handler will\nnot handle the exception.)\n\nBefore an except clause\'s suite is executed, details about the\nexception are assigned to three variables in the ``sys`` module:\n``sys.exc_type`` receives the object identifying the exception;\n``sys.exc_value`` receives the exception\'s parameter;\n``sys.exc_traceback`` receives a traceback object (see section *The\nstandard type hierarchy*) identifying the point in the program where\nthe exception occurred. These details are also available through the\n``sys.exc_info()`` function, which returns a tuple ``(exc_type,\nexc_value, exc_traceback)``. Use of the corresponding variables is\ndeprecated in favor of this function, since their use is unsafe in a\nthreaded program. As of Python 1.5, the variables are restored to\ntheir previous values (before the call) when returning from a function\nthat handled an exception.\n\nThe optional ``else`` clause is executed if and when control flows off\nthe end of the ``try`` clause. [2] Exceptions in the ``else`` clause\nare not handled by the preceding ``except`` clauses.\n\nIf ``finally`` is present, it specifies a \'cleanup\' handler. The\n``try`` clause is executed, including any ``except`` and ``else``\nclauses. If an exception occurs in any of the clauses and is not\nhandled, the exception is temporarily saved. The ``finally`` clause is\nexecuted. If there is a saved exception, it is re-raised at the end\nof the ``finally`` clause. If the ``finally`` clause raises another\nexception or executes a ``return`` or ``break`` statement, the saved\nexception is discarded:\n\n def f():\n try:\n 1/0\n finally:\n return 42\n\n >>> f()\n 42\n\nThe exception information is not available to the program during\nexecution of the ``finally`` clause.\n\nWhen a ``return``, ``break`` or ``continue`` statement is executed in\nthe ``try`` suite of a ``try``...``finally`` statement, the\n``finally`` clause is also executed \'on the way out.\' A ``continue``\nstatement is illegal in the ``finally`` clause. (The reason is a\nproblem with the current implementation --- this restriction may be\nlifted in the future).\n\nAdditional information on exceptions can be found in section\n*Exceptions*, and information on using the ``raise`` statement to\ngenerate exceptions may be found in section *The raise statement*.\n', - 'types': '\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python. Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types. Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.).\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\' These are attributes that provide access to the\nimplementation and are not intended for general use. Their definition\nmay change in the future.\n\nNone\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name ``None``.\n It is used to signify the absence of a value in many situations,\n e.g., it is returned from functions that don\'t explicitly return\n anything. Its truth value is false.\n\nNotImplemented\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``NotImplemented``. Numeric methods and rich comparison methods may\n return this value if they do not implement the operation for the\n operands provided. (The interpreter will then try the reflected\n operation, or some other fallback, depending on the operator.) Its\n truth value is true.\n\nEllipsis\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``Ellipsis``. It is used to indicate the presence of the ``...``\n syntax in a slice. Its truth value is true.\n\n``numbers.Number``\n These are created by numeric literals and returned as results by\n arithmetic operators and arithmetic built-in functions. Numeric\n objects are immutable; once created their value never changes.\n Python numbers are of course strongly related to mathematical\n numbers, but subject to the limitations of numerical representation\n in computers.\n\n Python distinguishes between integers, floating point numbers, and\n complex numbers:\n\n ``numbers.Integral``\n These represent elements from the mathematical set of integers\n (positive and negative).\n\n There are three types of integers:\n\n Plain integers\n These represent numbers in the range -2147483648 through\n 2147483647. (The range may be larger on machines with a\n larger natural word size, but not smaller.) When the result\n of an operation would fall outside this range, the result is\n normally returned as a long integer (in some cases, the\n exception ``OverflowError`` is raised instead). For the\n purpose of shift and mask operations, integers are assumed to\n have a binary, 2\'s complement notation using 32 or more bits,\n and hiding no bits from the user (i.e., all 4294967296\n different bit patterns correspond to different values).\n\n Long integers\n These represent numbers in an unlimited range, subject to\n available (virtual) memory only. For the purpose of shift\n and mask operations, a binary representation is assumed, and\n negative numbers are represented in a variant of 2\'s\n complement which gives the illusion of an infinite string of\n sign bits extending to the left.\n\n Booleans\n These represent the truth values False and True. The two\n objects representing the values False and True are the only\n Boolean objects. The Boolean type is a subtype of plain\n integers, and Boolean values behave like the values 0 and 1,\n respectively, in almost all contexts, the exception being\n that when converted to a string, the strings ``"False"`` or\n ``"True"`` are returned, respectively.\n\n The rules for integer representation are intended to give the\n most meaningful interpretation of shift and mask operations\n involving negative integers and the least surprises when\n switching between the plain and long integer domains. Any\n operation, if it yields a result in the plain integer domain,\n will yield the same result in the long integer domain or when\n using mixed operands. The switch between domains is transparent\n to the programmer.\n\n ``numbers.Real`` (``float``)\n These represent machine-level double precision floating point\n numbers. You are at the mercy of the underlying machine\n architecture (and C or Java implementation) for the accepted\n range and handling of overflow. Python does not support single-\n precision floating point numbers; the savings in processor and\n memory usage that are usually the reason for using these is\n dwarfed by the overhead of using objects in Python, so there is\n no reason to complicate the language with two kinds of floating\n point numbers.\n\n ``numbers.Complex``\n These represent complex numbers as a pair of machine-level\n double precision floating point numbers. The same caveats apply\n as for floating point numbers. The real and imaginary parts of a\n complex number ``z`` can be retrieved through the read-only\n attributes ``z.real`` and ``z.imag``.\n\nSequences\n These represent finite ordered sets indexed by non-negative\n numbers. The built-in function ``len()`` returns the number of\n items of a sequence. When the length of a sequence is *n*, the\n index set contains the numbers 0, 1, ..., *n*-1. Item *i* of\n sequence *a* is selected by ``a[i]``.\n\n Sequences also support slicing: ``a[i:j]`` selects all items with\n index *k* such that *i* ``<=`` *k* ``<`` *j*. When used as an\n expression, a slice is a sequence of the same type. This implies\n that the index set is renumbered so that it starts at 0.\n\n Some sequences also support "extended slicing" with a third "step"\n parameter: ``a[i:j:k]`` selects all items of *a* with index *x*\n where ``x = i + n*k``, *n* ``>=`` ``0`` and *i* ``<=`` *x* ``<``\n *j*.\n\n Sequences are distinguished according to their mutability:\n\n Immutable sequences\n An object of an immutable sequence type cannot change once it is\n created. (If the object contains references to other objects,\n these other objects may be mutable and may be changed; however,\n the collection of objects directly referenced by an immutable\n object cannot change.)\n\n The following types are immutable sequences:\n\n Strings\n The items of a string are characters. There is no separate\n character type; a character is represented by a string of one\n item. Characters represent (at least) 8-bit bytes. The\n built-in functions ``chr()`` and ``ord()`` convert between\n characters and nonnegative integers representing the byte\n values. Bytes with the values 0-127 usually represent the\n corresponding ASCII values, but the interpretation of values\n is up to the program. The string data type is also used to\n represent arrays of bytes, e.g., to hold data read from a\n file.\n\n (On systems whose native character set is not ASCII, strings\n may use EBCDIC in their internal representation, provided the\n functions ``chr()`` and ``ord()`` implement a mapping between\n ASCII and EBCDIC, and string comparison preserves the ASCII\n order. Or perhaps someone can propose a better rule?)\n\n Unicode\n The items of a Unicode object are Unicode code units. A\n Unicode code unit is represented by a Unicode object of one\n item and can hold either a 16-bit or 32-bit value\n representing a Unicode ordinal (the maximum value for the\n ordinal is given in ``sys.maxunicode``, and depends on how\n Python is configured at compile time). Surrogate pairs may\n be present in the Unicode object, and will be reported as two\n separate items. The built-in functions ``unichr()`` and\n ``ord()`` convert between code units and nonnegative integers\n representing the Unicode ordinals as defined in the Unicode\n Standard 3.0. Conversion from and to other encodings are\n possible through the Unicode method ``encode()`` and the\n built-in function ``unicode()``.\n\n Tuples\n The items of a tuple are arbitrary Python objects. Tuples of\n two or more items are formed by comma-separated lists of\n expressions. A tuple of one item (a \'singleton\') can be\n formed by affixing a comma to an expression (an expression by\n itself does not create a tuple, since parentheses must be\n usable for grouping of expressions). An empty tuple can be\n formed by an empty pair of parentheses.\n\n Mutable sequences\n Mutable sequences can be changed after they are created. The\n subscription and slicing notations can be used as the target of\n assignment and ``del`` (delete) statements.\n\n There are currently two intrinsic mutable sequence types:\n\n Lists\n The items of a list are arbitrary Python objects. Lists are\n formed by placing a comma-separated list of expressions in\n square brackets. (Note that there are no special cases needed\n to form lists of length 0 or 1.)\n\n Byte Arrays\n A bytearray object is a mutable array. They are created by\n the built-in ``bytearray()`` constructor. Aside from being\n mutable (and hence unhashable), byte arrays otherwise provide\n the same interface and functionality as immutable bytes\n objects.\n\n The extension module ``array`` provides an additional example of\n a mutable sequence type.\n\nSet types\n These represent unordered, finite sets of unique, immutable\n objects. As such, they cannot be indexed by any subscript. However,\n they can be iterated over, and the built-in function ``len()``\n returns the number of items in a set. Common uses for sets are fast\n membership testing, removing duplicates from a sequence, and\n computing mathematical operations such as intersection, union,\n difference, and symmetric difference.\n\n For set elements, the same immutability rules apply as for\n dictionary keys. Note that numeric types obey the normal rules for\n numeric comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``), only one of them can be contained in a set.\n\n There are currently two intrinsic set types:\n\n Sets\n These represent a mutable set. They are created by the built-in\n ``set()`` constructor and can be modified afterwards by several\n methods, such as ``add()``.\n\n Frozen sets\n These represent an immutable set. They are created by the\n built-in ``frozenset()`` constructor. As a frozenset is\n immutable and *hashable*, it can be used again as an element of\n another set, or as a dictionary key.\n\nMappings\n These represent finite sets of objects indexed by arbitrary index\n sets. The subscript notation ``a[k]`` selects the item indexed by\n ``k`` from the mapping ``a``; this can be used in expressions and\n as the target of assignments or ``del`` statements. The built-in\n function ``len()`` returns the number of items in a mapping.\n\n There is currently a single intrinsic mapping type:\n\n Dictionaries\n These represent finite sets of objects indexed by nearly\n arbitrary values. The only types of values not acceptable as\n keys are values containing lists or dictionaries or other\n mutable types that are compared by value rather than by object\n identity, the reason being that the efficient implementation of\n dictionaries requires a key\'s hash value to remain constant.\n Numeric types used for keys obey the normal rules for numeric\n comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``) then they can be used interchangeably to index the same\n dictionary entry.\n\n Dictionaries are mutable; they can be created by the ``{...}``\n notation (see section *Dictionary displays*).\n\n The extension modules ``dbm``, ``gdbm``, and ``bsddb`` provide\n additional examples of mapping types.\n\nCallable types\n These are the types to which the function call operation (see\n section *Calls*) can be applied:\n\n User-defined functions\n A user-defined function object is created by a function\n definition (see section *Function definitions*). It should be\n called with an argument list containing the same number of items\n as the function\'s formal parameter list.\n\n Special attributes:\n\n +-------------------------+---------------------------------+-------------+\n | Attribute | Meaning | |\n +=========================+=================================+=============+\n | ``func_doc`` | The function\'s documentation | Writable |\n | | string, or ``None`` if | |\n | | unavailable | |\n +-------------------------+---------------------------------+-------------+\n | ``__doc__`` | Another way of spelling | Writable |\n | | ``func_doc`` | |\n +-------------------------+---------------------------------+-------------+\n | ``func_name`` | The function\'s name | Writable |\n +-------------------------+---------------------------------+-------------+\n | ``__name__`` | Another way of spelling | Writable |\n | | ``func_name`` | |\n +-------------------------+---------------------------------+-------------+\n | ``__module__`` | The name of the module the | Writable |\n | | function was defined in, or | |\n | | ``None`` if unavailable. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_defaults`` | A tuple containing default | Writable |\n | | argument values for those | |\n | | arguments that have defaults, | |\n | | or ``None`` if no arguments | |\n | | have a default value | |\n +-------------------------+---------------------------------+-------------+\n | ``func_code`` | The code object representing | Writable |\n | | the compiled function body. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_globals`` | A reference to the dictionary | Read-only |\n | | that holds the function\'s | |\n | | global variables --- the global | |\n | | namespace of the module in | |\n | | which the function was defined. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_dict`` | The namespace supporting | Writable |\n | | arbitrary function attributes. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_closure`` | ``None`` or a tuple of cells | Read-only |\n | | that contain bindings for the | |\n | | function\'s free variables. | |\n +-------------------------+---------------------------------+-------------+\n\n Most of the attributes labelled "Writable" check the type of the\n assigned value.\n\n Changed in version 2.4: ``func_name`` is now writable.\n\n Function objects also support getting and setting arbitrary\n attributes, which can be used, for example, to attach metadata\n to functions. Regular attribute dot-notation is used to get and\n set such attributes. *Note that the current implementation only\n supports function attributes on user-defined functions. Function\n attributes on built-in functions may be supported in the\n future.*\n\n Additional information about a function\'s definition can be\n retrieved from its code object; see the description of internal\n types below.\n\n User-defined methods\n A user-defined method object combines a class, a class instance\n (or ``None``) and any callable object (normally a user-defined\n function).\n\n Special read-only attributes: ``im_self`` is the class instance\n object, ``im_func`` is the function object; ``im_class`` is the\n class of ``im_self`` for bound methods or the class that asked\n for the method for unbound methods; ``__doc__`` is the method\'s\n documentation (same as ``im_func.__doc__``); ``__name__`` is the\n method name (same as ``im_func.__name__``); ``__module__`` is\n the name of the module the method was defined in, or ``None`` if\n unavailable.\n\n Changed in version 2.2: ``im_self`` used to refer to the class\n that defined the method.\n\n Changed in version 2.6: For Python 3 forward-compatibility,\n ``im_func`` is also available as ``__func__``, and ``im_self``\n as ``__self__``.\n\n Methods also support accessing (but not setting) the arbitrary\n function attributes on the underlying function object.\n\n User-defined method objects may be created when getting an\n attribute of a class (perhaps via an instance of that class), if\n that attribute is a user-defined function object, an unbound\n user-defined method object, or a class method object. When the\n attribute is a user-defined method object, a new method object\n is only created if the class from which it is being retrieved is\n the same as, or a derived class of, the class stored in the\n original method object; otherwise, the original method object is\n used as it is.\n\n When a user-defined method object is created by retrieving a\n user-defined function object from a class, its ``im_self``\n attribute is ``None`` and the method object is said to be\n unbound. When one is created by retrieving a user-defined\n function object from a class via one of its instances, its\n ``im_self`` attribute is the instance, and the method object is\n said to be bound. In either case, the new method\'s ``im_class``\n attribute is the class from which the retrieval takes place, and\n its ``im_func`` attribute is the original function object.\n\n When a user-defined method object is created by retrieving\n another method object from a class or instance, the behaviour is\n the same as for a function object, except that the ``im_func``\n attribute of the new instance is not the original method object\n but its ``im_func`` attribute.\n\n When a user-defined method object is created by retrieving a\n class method object from a class or instance, its ``im_self``\n attribute is the class itself (the same as the ``im_class``\n attribute), and its ``im_func`` attribute is the function object\n underlying the class method.\n\n When an unbound user-defined method object is called, the\n underlying function (``im_func``) is called, with the\n restriction that the first argument must be an instance of the\n proper class (``im_class``) or of a derived class thereof.\n\n When a bound user-defined method object is called, the\n underlying function (``im_func``) is called, inserting the class\n instance (``im_self``) in front of the argument list. For\n instance, when ``C`` is a class which contains a definition for\n a function ``f()``, and ``x`` is an instance of ``C``, calling\n ``x.f(1)`` is equivalent to calling ``C.f(x, 1)``.\n\n When a user-defined method object is derived from a class method\n object, the "class instance" stored in ``im_self`` will actually\n be the class itself, so that calling either ``x.f(1)`` or\n ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is\n the underlying function.\n\n Note that the transformation from function object to (unbound or\n bound) method object happens each time the attribute is\n retrieved from the class or instance. In some cases, a fruitful\n optimization is to assign the attribute to a local variable and\n call that local variable. Also notice that this transformation\n only happens for user-defined functions; other callable objects\n (and all non-callable objects) are retrieved without\n transformation. It is also important to note that user-defined\n functions which are attributes of a class instance are not\n converted to bound methods; this *only* happens when the\n function is an attribute of the class.\n\n Generator functions\n A function or method which uses the ``yield`` statement (see\n section *The yield statement*) is called a *generator function*.\n Such a function, when called, always returns an iterator object\n which can be used to execute the body of the function: calling\n the iterator\'s ``next()`` method will cause the function to\n execute until it provides a value using the ``yield`` statement.\n When the function executes a ``return`` statement or falls off\n the end, a ``StopIteration`` exception is raised and the\n iterator will have reached the end of the set of values to be\n returned.\n\n Built-in functions\n A built-in function object is a wrapper around a C function.\n Examples of built-in functions are ``len()`` and ``math.sin()``\n (``math`` is a standard built-in module). The number and type of\n the arguments are determined by the C function. Special read-\n only attributes: ``__doc__`` is the function\'s documentation\n string, or ``None`` if unavailable; ``__name__`` is the\n function\'s name; ``__self__`` is set to ``None`` (but see the\n next item); ``__module__`` is the name of the module the\n function was defined in or ``None`` if unavailable.\n\n Built-in methods\n This is really a different disguise of a built-in function, this\n time containing an object passed to the C function as an\n implicit extra argument. An example of a built-in method is\n ``alist.append()``, assuming *alist* is a list object. In this\n case, the special read-only attribute ``__self__`` is set to the\n object denoted by *alist*.\n\n Class Types\n Class types, or "new-style classes," are callable. These\n objects normally act as factories for new instances of\n themselves, but variations are possible for class types that\n override ``__new__()``. The arguments of the call are passed to\n ``__new__()`` and, in the typical case, to ``__init__()`` to\n initialize the new instance.\n\n Classic Classes\n Class objects are described below. When a class object is\n called, a new class instance (also described below) is created\n and returned. This implies a call to the class\'s ``__init__()``\n method if it has one. Any arguments are passed on to the\n ``__init__()`` method. If there is no ``__init__()`` method,\n the class must be called without arguments.\n\n Class instances\n Class instances are described below. Class instances are\n callable only when the class has a ``__call__()`` method;\n ``x(arguments)`` is a shorthand for ``x.__call__(arguments)``.\n\nModules\n Modules are imported by the ``import`` statement (see section *The\n import statement*). A module object has a namespace implemented by\n a dictionary object (this is the dictionary referenced by the\n func_globals attribute of functions defined in the module).\n Attribute references are translated to lookups in this dictionary,\n e.g., ``m.x`` is equivalent to ``m.__dict__["x"]``. A module object\n does not contain the code object used to initialize the module\n (since it isn\'t needed once the initialization is done).\n\n Attribute assignment updates the module\'s namespace dictionary,\n e.g., ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``.\n\n Special read-only attribute: ``__dict__`` is the module\'s namespace\n as a dictionary object.\n\n **CPython implementation detail:** Because of the way CPython\n clears module dictionaries, the module dictionary will be cleared\n when the module falls out of scope even if the dictionary still has\n live references. To avoid this, copy the dictionary or keep the\n module around while using its dictionary directly.\n\n Predefined (writable) attributes: ``__name__`` is the module\'s\n name; ``__doc__`` is the module\'s documentation string, or ``None``\n if unavailable; ``__file__`` is the pathname of the file from which\n the module was loaded, if it was loaded from a file. The\n ``__file__`` attribute is not present for C modules that are\n statically linked into the interpreter; for extension modules\n loaded dynamically from a shared library, it is the pathname of the\n shared library file.\n\nClasses\n Both class types (new-style classes) and class objects (old-\n style/classic classes) are typically created by class definitions\n (see section *Class definitions*). A class has a namespace\n implemented by a dictionary object. Class attribute references are\n translated to lookups in this dictionary, e.g., ``C.x`` is\n translated to ``C.__dict__["x"]`` (although for new-style classes\n in particular there are a number of hooks which allow for other\n means of locating attributes). When the attribute name is not found\n there, the attribute search continues in the base classes. For\n old-style classes, the search is depth-first, left-to-right in the\n order of occurrence in the base class list. New-style classes use\n the more complex C3 method resolution order which behaves correctly\n even in the presence of \'diamond\' inheritance structures where\n there are multiple inheritance paths leading back to a common\n ancestor. Additional details on the C3 MRO used by new-style\n classes can be found in the documentation accompanying the 2.3\n release at http://www.python.org/download/releases/2.3/mro/.\n\n When a class attribute reference (for class ``C``, say) would yield\n a user-defined function object or an unbound user-defined method\n object whose associated class is either ``C`` or one of its base\n classes, it is transformed into an unbound user-defined method\n object whose ``im_class`` attribute is ``C``. When it would yield a\n class method object, it is transformed into a bound user-defined\n method object whose ``im_class`` and ``im_self`` attributes are\n both ``C``. When it would yield a static method object, it is\n transformed into the object wrapped by the static method object.\n See section *Implementing Descriptors* for another way in which\n attributes retrieved from a class may differ from those actually\n contained in its ``__dict__`` (note that only new-style classes\n support descriptors).\n\n Class attribute assignments update the class\'s dictionary, never\n the dictionary of a base class.\n\n A class object can be called (see above) to yield a class instance\n (see below).\n\n Special attributes: ``__name__`` is the class name; ``__module__``\n is the module name in which the class was defined; ``__dict__`` is\n the dictionary containing the class\'s namespace; ``__bases__`` is a\n tuple (possibly empty or a singleton) containing the base classes,\n in the order of their occurrence in the base class list;\n ``__doc__`` is the class\'s documentation string, or None if\n undefined.\n\nClass instances\n A class instance is created by calling a class object (see above).\n A class instance has a namespace implemented as a dictionary which\n is the first place in which attribute references are searched.\n When an attribute is not found there, and the instance\'s class has\n an attribute by that name, the search continues with the class\n attributes. If a class attribute is found that is a user-defined\n function object or an unbound user-defined method object whose\n associated class is the class (call it ``C``) of the instance for\n which the attribute reference was initiated or one of its bases, it\n is transformed into a bound user-defined method object whose\n ``im_class`` attribute is ``C`` and whose ``im_self`` attribute is\n the instance. Static method and class method objects are also\n transformed, as if they had been retrieved from class ``C``; see\n above under "Classes". See section *Implementing Descriptors* for\n another way in which attributes of a class retrieved via its\n instances may differ from the objects actually stored in the\n class\'s ``__dict__``. If no class attribute is found, and the\n object\'s class has a ``__getattr__()`` method, that is called to\n satisfy the lookup.\n\n Attribute assignments and deletions update the instance\'s\n dictionary, never a class\'s dictionary. If the class has a\n ``__setattr__()`` or ``__delattr__()`` method, this is called\n instead of updating the instance dictionary directly.\n\n Class instances can pretend to be numbers, sequences, or mappings\n if they have methods with certain special names. See section\n *Special method names*.\n\n Special attributes: ``__dict__`` is the attribute dictionary;\n ``__class__`` is the instance\'s class.\n\nFiles\n A file object represents an open file. File objects are created by\n the ``open()`` built-in function, and also by ``os.popen()``,\n ``os.fdopen()``, and the ``makefile()`` method of socket objects\n (and perhaps by other functions or methods provided by extension\n modules). The objects ``sys.stdin``, ``sys.stdout`` and\n ``sys.stderr`` are initialized to file objects corresponding to the\n interpreter\'s standard input, output and error streams. See *File\n Objects* for complete documentation of file objects.\n\nInternal types\n A few types used internally by the interpreter are exposed to the\n user. Their definitions may change with future versions of the\n interpreter, but they are mentioned here for completeness.\n\n Code objects\n Code objects represent *byte-compiled* executable Python code,\n or *bytecode*. The difference between a code object and a\n function object is that the function object contains an explicit\n reference to the function\'s globals (the module in which it was\n defined), while a code object contains no context; also the\n default argument values are stored in the function object, not\n in the code object (because they represent values calculated at\n run-time). Unlike function objects, code objects are immutable\n and contain no references (directly or indirectly) to mutable\n objects.\n\n Special read-only attributes: ``co_name`` gives the function\n name; ``co_argcount`` is the number of positional arguments\n (including arguments with default values); ``co_nlocals`` is the\n number of local variables used by the function (including\n arguments); ``co_varnames`` is a tuple containing the names of\n the local variables (starting with the argument names);\n ``co_cellvars`` is a tuple containing the names of local\n variables that are referenced by nested functions;\n ``co_freevars`` is a tuple containing the names of free\n variables; ``co_code`` is a string representing the sequence of\n bytecode instructions; ``co_consts`` is a tuple containing the\n literals used by the bytecode; ``co_names`` is a tuple\n containing the names used by the bytecode; ``co_filename`` is\n the filename from which the code was compiled;\n ``co_firstlineno`` is the first line number of the function;\n ``co_lnotab`` is a string encoding the mapping from bytecode\n offsets to line numbers (for details see the source code of the\n interpreter); ``co_stacksize`` is the required stack size\n (including local variables); ``co_flags`` is an integer encoding\n a number of flags for the interpreter.\n\n The following flag bits are defined for ``co_flags``: bit\n ``0x04`` is set if the function uses the ``*arguments`` syntax\n to accept an arbitrary number of positional arguments; bit\n ``0x08`` is set if the function uses the ``**keywords`` syntax\n to accept arbitrary keyword arguments; bit ``0x20`` is set if\n the function is a generator.\n\n Future feature declarations (``from __future__ import\n division``) also use bits in ``co_flags`` to indicate whether a\n code object was compiled with a particular feature enabled: bit\n ``0x2000`` is set if the function was compiled with future\n division enabled; bits ``0x10`` and ``0x1000`` were used in\n earlier versions of Python.\n\n Other bits in ``co_flags`` are reserved for internal use.\n\n If a code object represents a function, the first item in\n ``co_consts`` is the documentation string of the function, or\n ``None`` if undefined.\n\n Frame objects\n Frame objects represent execution frames. They may occur in\n traceback objects (see below).\n\n Special read-only attributes: ``f_back`` is to the previous\n stack frame (towards the caller), or ``None`` if this is the\n bottom stack frame; ``f_code`` is the code object being executed\n in this frame; ``f_locals`` is the dictionary used to look up\n local variables; ``f_globals`` is used for global variables;\n ``f_builtins`` is used for built-in (intrinsic) names;\n ``f_restricted`` is a flag indicating whether the function is\n executing in restricted execution mode; ``f_lasti`` gives the\n precise instruction (this is an index into the bytecode string\n of the code object).\n\n Special writable attributes: ``f_trace``, if not ``None``, is a\n function called at the start of each source code line (this is\n used by the debugger); ``f_exc_type``, ``f_exc_value``,\n ``f_exc_traceback`` represent the last exception raised in the\n parent frame provided another exception was ever raised in the\n current frame (in all other cases they are None); ``f_lineno``\n is the current line number of the frame --- writing to this from\n within a trace function jumps to the given line (only for the\n bottom-most frame). A debugger can implement a Jump command\n (aka Set Next Statement) by writing to f_lineno.\n\n Traceback objects\n Traceback objects represent a stack trace of an exception. A\n traceback object is created when an exception occurs. When the\n search for an exception handler unwinds the execution stack, at\n each unwound level a traceback object is inserted in front of\n the current traceback. When an exception handler is entered,\n the stack trace is made available to the program. (See section\n *The try statement*.) It is accessible as ``sys.exc_traceback``,\n and also as the third item of the tuple returned by\n ``sys.exc_info()``. The latter is the preferred interface,\n since it works correctly when the program is using multiple\n threads. When the program contains no suitable handler, the\n stack trace is written (nicely formatted) to the standard error\n stream; if the interpreter is interactive, it is also made\n available to the user as ``sys.last_traceback``.\n\n Special read-only attributes: ``tb_next`` is the next level in\n the stack trace (towards the frame where the exception\n occurred), or ``None`` if there is no next level; ``tb_frame``\n points to the execution frame of the current level;\n ``tb_lineno`` gives the line number where the exception\n occurred; ``tb_lasti`` indicates the precise instruction. The\n line number and last instruction in the traceback may differ\n from the line number of its frame object if the exception\n occurred in a ``try`` statement with no matching except clause\n or with a finally clause.\n\n Slice objects\n Slice objects are used to represent slices when *extended slice\n syntax* is used. This is a slice using two colons, or multiple\n slices or ellipses separated by commas, e.g., ``a[i:j:step]``,\n ``a[i:j, k:l]``, or ``a[..., i:j]``. They are also created by\n the built-in ``slice()`` function.\n\n Special read-only attributes: ``start`` is the lower bound;\n ``stop`` is the upper bound; ``step`` is the step value; each is\n ``None`` if omitted. These attributes can have any type.\n\n Slice objects support one method:\n\n slice.indices(self, length)\n\n This method takes a single integer argument *length* and\n computes information about the extended slice that the slice\n object would describe if applied to a sequence of *length*\n items. It returns a tuple of three integers; respectively\n these are the *start* and *stop* indices and the *step* or\n stride length of the slice. Missing or out-of-bounds indices\n are handled in a manner consistent with regular slices.\n\n New in version 2.3.\n\n Static method objects\n Static method objects provide a way of defeating the\n transformation of function objects to method objects described\n above. A static method object is a wrapper around any other\n object, usually a user-defined method object. When a static\n method object is retrieved from a class or a class instance, the\n object actually returned is the wrapped object, which is not\n subject to any further transformation. Static method objects are\n not themselves callable, although the objects they wrap usually\n are. Static method objects are created by the built-in\n ``staticmethod()`` constructor.\n\n Class method objects\n A class method object, like a static method object, is a wrapper\n around another object that alters the way in which that object\n is retrieved from classes and class instances. The behaviour of\n class method objects upon such retrieval is described above,\n under "User-defined methods". Class method objects are created\n by the built-in ``classmethod()`` constructor.\n', + 'types': '\nThe standard type hierarchy\n***************************\n\nBelow is a list of the types that are built into Python. Extension\nmodules (written in C, Java, or other languages, depending on the\nimplementation) can define additional types. Future versions of\nPython may add types to the type hierarchy (e.g., rational numbers,\nefficiently stored arrays of integers, etc.).\n\nSome of the type descriptions below contain a paragraph listing\n\'special attributes.\' These are attributes that provide access to the\nimplementation and are not intended for general use. Their definition\nmay change in the future.\n\nNone\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name ``None``.\n It is used to signify the absence of a value in many situations,\n e.g., it is returned from functions that don\'t explicitly return\n anything. Its truth value is false.\n\nNotImplemented\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``NotImplemented``. Numeric methods and rich comparison methods may\n return this value if they do not implement the operation for the\n operands provided. (The interpreter will then try the reflected\n operation, or some other fallback, depending on the operator.) Its\n truth value is true.\n\nEllipsis\n This type has a single value. There is a single object with this\n value. This object is accessed through the built-in name\n ``Ellipsis``. It is used to indicate the presence of the ``...``\n syntax in a slice. Its truth value is true.\n\n``numbers.Number``\n These are created by numeric literals and returned as results by\n arithmetic operators and arithmetic built-in functions. Numeric\n objects are immutable; once created their value never changes.\n Python numbers are of course strongly related to mathematical\n numbers, but subject to the limitations of numerical representation\n in computers.\n\n Python distinguishes between integers, floating point numbers, and\n complex numbers:\n\n ``numbers.Integral``\n These represent elements from the mathematical set of integers\n (positive and negative).\n\n There are three types of integers:\n\n Plain integers\n These represent numbers in the range -2147483648 through\n 2147483647. (The range may be larger on machines with a\n larger natural word size, but not smaller.) When the result\n of an operation would fall outside this range, the result is\n normally returned as a long integer (in some cases, the\n exception ``OverflowError`` is raised instead). For the\n purpose of shift and mask operations, integers are assumed to\n have a binary, 2\'s complement notation using 32 or more bits,\n and hiding no bits from the user (i.e., all 4294967296\n different bit patterns correspond to different values).\n\n Long integers\n These represent numbers in an unlimited range, subject to\n available (virtual) memory only. For the purpose of shift\n and mask operations, a binary representation is assumed, and\n negative numbers are represented in a variant of 2\'s\n complement which gives the illusion of an infinite string of\n sign bits extending to the left.\n\n Booleans\n These represent the truth values False and True. The two\n objects representing the values False and True are the only\n Boolean objects. The Boolean type is a subtype of plain\n integers, and Boolean values behave like the values 0 and 1,\n respectively, in almost all contexts, the exception being\n that when converted to a string, the strings ``"False"`` or\n ``"True"`` are returned, respectively.\n\n The rules for integer representation are intended to give the\n most meaningful interpretation of shift and mask operations\n involving negative integers and the least surprises when\n switching between the plain and long integer domains. Any\n operation, if it yields a result in the plain integer domain,\n will yield the same result in the long integer domain or when\n using mixed operands. The switch between domains is transparent\n to the programmer.\n\n ``numbers.Real`` (``float``)\n These represent machine-level double precision floating point\n numbers. You are at the mercy of the underlying machine\n architecture (and C or Java implementation) for the accepted\n range and handling of overflow. Python does not support single-\n precision floating point numbers; the savings in processor and\n memory usage that are usually the reason for using these is\n dwarfed by the overhead of using objects in Python, so there is\n no reason to complicate the language with two kinds of floating\n point numbers.\n\n ``numbers.Complex``\n These represent complex numbers as a pair of machine-level\n double precision floating point numbers. The same caveats apply\n as for floating point numbers. The real and imaginary parts of a\n complex number ``z`` can be retrieved through the read-only\n attributes ``z.real`` and ``z.imag``.\n\nSequences\n These represent finite ordered sets indexed by non-negative\n numbers. The built-in function ``len()`` returns the number of\n items of a sequence. When the length of a sequence is *n*, the\n index set contains the numbers 0, 1, ..., *n*-1. Item *i* of\n sequence *a* is selected by ``a[i]``.\n\n Sequences also support slicing: ``a[i:j]`` selects all items with\n index *k* such that *i* ``<=`` *k* ``<`` *j*. When used as an\n expression, a slice is a sequence of the same type. This implies\n that the index set is renumbered so that it starts at 0.\n\n Some sequences also support "extended slicing" with a third "step"\n parameter: ``a[i:j:k]`` selects all items of *a* with index *x*\n where ``x = i + n*k``, *n* ``>=`` ``0`` and *i* ``<=`` *x* ``<``\n *j*.\n\n Sequences are distinguished according to their mutability:\n\n Immutable sequences\n An object of an immutable sequence type cannot change once it is\n created. (If the object contains references to other objects,\n these other objects may be mutable and may be changed; however,\n the collection of objects directly referenced by an immutable\n object cannot change.)\n\n The following types are immutable sequences:\n\n Strings\n The items of a string are characters. There is no separate\n character type; a character is represented by a string of one\n item. Characters represent (at least) 8-bit bytes. The\n built-in functions ``chr()`` and ``ord()`` convert between\n characters and nonnegative integers representing the byte\n values. Bytes with the values 0-127 usually represent the\n corresponding ASCII values, but the interpretation of values\n is up to the program. The string data type is also used to\n represent arrays of bytes, e.g., to hold data read from a\n file.\n\n (On systems whose native character set is not ASCII, strings\n may use EBCDIC in their internal representation, provided the\n functions ``chr()`` and ``ord()`` implement a mapping between\n ASCII and EBCDIC, and string comparison preserves the ASCII\n order. Or perhaps someone can propose a better rule?)\n\n Unicode\n The items of a Unicode object are Unicode code units. A\n Unicode code unit is represented by a Unicode object of one\n item and can hold either a 16-bit or 32-bit value\n representing a Unicode ordinal (the maximum value for the\n ordinal is given in ``sys.maxunicode``, and depends on how\n Python is configured at compile time). Surrogate pairs may\n be present in the Unicode object, and will be reported as two\n separate items. The built-in functions ``unichr()`` and\n ``ord()`` convert between code units and nonnegative integers\n representing the Unicode ordinals as defined in the Unicode\n Standard 3.0. Conversion from and to other encodings are\n possible through the Unicode method ``encode()`` and the\n built-in function ``unicode()``.\n\n Tuples\n The items of a tuple are arbitrary Python objects. Tuples of\n two or more items are formed by comma-separated lists of\n expressions. A tuple of one item (a \'singleton\') can be\n formed by affixing a comma to an expression (an expression by\n itself does not create a tuple, since parentheses must be\n usable for grouping of expressions). An empty tuple can be\n formed by an empty pair of parentheses.\n\n Mutable sequences\n Mutable sequences can be changed after they are created. The\n subscription and slicing notations can be used as the target of\n assignment and ``del`` (delete) statements.\n\n There are currently two intrinsic mutable sequence types:\n\n Lists\n The items of a list are arbitrary Python objects. Lists are\n formed by placing a comma-separated list of expressions in\n square brackets. (Note that there are no special cases needed\n to form lists of length 0 or 1.)\n\n Byte Arrays\n A bytearray object is a mutable array. They are created by\n the built-in ``bytearray()`` constructor. Aside from being\n mutable (and hence unhashable), byte arrays otherwise provide\n the same interface and functionality as immutable bytes\n objects.\n\n The extension module ``array`` provides an additional example of\n a mutable sequence type.\n\nSet types\n These represent unordered, finite sets of unique, immutable\n objects. As such, they cannot be indexed by any subscript. However,\n they can be iterated over, and the built-in function ``len()``\n returns the number of items in a set. Common uses for sets are fast\n membership testing, removing duplicates from a sequence, and\n computing mathematical operations such as intersection, union,\n difference, and symmetric difference.\n\n For set elements, the same immutability rules apply as for\n dictionary keys. Note that numeric types obey the normal rules for\n numeric comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``), only one of them can be contained in a set.\n\n There are currently two intrinsic set types:\n\n Sets\n These represent a mutable set. They are created by the built-in\n ``set()`` constructor and can be modified afterwards by several\n methods, such as ``add()``.\n\n Frozen sets\n These represent an immutable set. They are created by the\n built-in ``frozenset()`` constructor. As a frozenset is\n immutable and *hashable*, it can be used again as an element of\n another set, or as a dictionary key.\n\nMappings\n These represent finite sets of objects indexed by arbitrary index\n sets. The subscript notation ``a[k]`` selects the item indexed by\n ``k`` from the mapping ``a``; this can be used in expressions and\n as the target of assignments or ``del`` statements. The built-in\n function ``len()`` returns the number of items in a mapping.\n\n There is currently a single intrinsic mapping type:\n\n Dictionaries\n These represent finite sets of objects indexed by nearly\n arbitrary values. The only types of values not acceptable as\n keys are values containing lists or dictionaries or other\n mutable types that are compared by value rather than by object\n identity, the reason being that the efficient implementation of\n dictionaries requires a key\'s hash value to remain constant.\n Numeric types used for keys obey the normal rules for numeric\n comparison: if two numbers compare equal (e.g., ``1`` and\n ``1.0``) then they can be used interchangeably to index the same\n dictionary entry.\n\n Dictionaries are mutable; they can be created by the ``{...}``\n notation (see section *Dictionary displays*).\n\n The extension modules ``dbm``, ``gdbm``, and ``bsddb`` provide\n additional examples of mapping types.\n\nCallable types\n These are the types to which the function call operation (see\n section *Calls*) can be applied:\n\n User-defined functions\n A user-defined function object is created by a function\n definition (see section *Function definitions*). It should be\n called with an argument list containing the same number of items\n as the function\'s formal parameter list.\n\n Special attributes:\n\n +-------------------------+---------------------------------+-------------+\n | Attribute | Meaning | |\n +=========================+=================================+=============+\n | ``func_doc`` | The function\'s documentation | Writable |\n | | string, or ``None`` if | |\n | | unavailable | |\n +-------------------------+---------------------------------+-------------+\n | ``__doc__`` | Another way of spelling | Writable |\n | | ``func_doc`` | |\n +-------------------------+---------------------------------+-------------+\n | ``func_name`` | The function\'s name | Writable |\n +-------------------------+---------------------------------+-------------+\n | ``__name__`` | Another way of spelling | Writable |\n | | ``func_name`` | |\n +-------------------------+---------------------------------+-------------+\n | ``__module__`` | The name of the module the | Writable |\n | | function was defined in, or | |\n | | ``None`` if unavailable. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_defaults`` | A tuple containing default | Writable |\n | | argument values for those | |\n | | arguments that have defaults, | |\n | | or ``None`` if no arguments | |\n | | have a default value | |\n +-------------------------+---------------------------------+-------------+\n | ``func_code`` | The code object representing | Writable |\n | | the compiled function body. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_globals`` | A reference to the dictionary | Read-only |\n | | that holds the function\'s | |\n | | global variables --- the global | |\n | | namespace of the module in | |\n | | which the function was defined. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_dict`` | The namespace supporting | Writable |\n | | arbitrary function attributes. | |\n +-------------------------+---------------------------------+-------------+\n | ``func_closure`` | ``None`` or a tuple of cells | Read-only |\n | | that contain bindings for the | |\n | | function\'s free variables. | |\n +-------------------------+---------------------------------+-------------+\n\n Most of the attributes labelled "Writable" check the type of the\n assigned value.\n\n Changed in version 2.4: ``func_name`` is now writable.\n\n Function objects also support getting and setting arbitrary\n attributes, which can be used, for example, to attach metadata\n to functions. Regular attribute dot-notation is used to get and\n set such attributes. *Note that the current implementation only\n supports function attributes on user-defined functions. Function\n attributes on built-in functions may be supported in the\n future.*\n\n Additional information about a function\'s definition can be\n retrieved from its code object; see the description of internal\n types below.\n\n User-defined methods\n A user-defined method object combines a class, a class instance\n (or ``None``) and any callable object (normally a user-defined\n function).\n\n Special read-only attributes: ``im_self`` is the class instance\n object, ``im_func`` is the function object; ``im_class`` is the\n class of ``im_self`` for bound methods or the class that asked\n for the method for unbound methods; ``__doc__`` is the method\'s\n documentation (same as ``im_func.__doc__``); ``__name__`` is the\n method name (same as ``im_func.__name__``); ``__module__`` is\n the name of the module the method was defined in, or ``None`` if\n unavailable.\n\n Changed in version 2.2: ``im_self`` used to refer to the class\n that defined the method.\n\n Changed in version 2.6: For Python 3 forward-compatibility,\n ``im_func`` is also available as ``__func__``, and ``im_self``\n as ``__self__``.\n\n Methods also support accessing (but not setting) the arbitrary\n function attributes on the underlying function object.\n\n User-defined method objects may be created when getting an\n attribute of a class (perhaps via an instance of that class), if\n that attribute is a user-defined function object, an unbound\n user-defined method object, or a class method object. When the\n attribute is a user-defined method object, a new method object\n is only created if the class from which it is being retrieved is\n the same as, or a derived class of, the class stored in the\n original method object; otherwise, the original method object is\n used as it is.\n\n When a user-defined method object is created by retrieving a\n user-defined function object from a class, its ``im_self``\n attribute is ``None`` and the method object is said to be\n unbound. When one is created by retrieving a user-defined\n function object from a class via one of its instances, its\n ``im_self`` attribute is the instance, and the method object is\n said to be bound. In either case, the new method\'s ``im_class``\n attribute is the class from which the retrieval takes place, and\n its ``im_func`` attribute is the original function object.\n\n When a user-defined method object is created by retrieving\n another method object from a class or instance, the behaviour is\n the same as for a function object, except that the ``im_func``\n attribute of the new instance is not the original method object\n but its ``im_func`` attribute.\n\n When a user-defined method object is created by retrieving a\n class method object from a class or instance, its ``im_self``\n attribute is the class itself, and its ``im_func`` attribute is\n the function object underlying the class method.\n\n When an unbound user-defined method object is called, the\n underlying function (``im_func``) is called, with the\n restriction that the first argument must be an instance of the\n proper class (``im_class``) or of a derived class thereof.\n\n When a bound user-defined method object is called, the\n underlying function (``im_func``) is called, inserting the class\n instance (``im_self``) in front of the argument list. For\n instance, when ``C`` is a class which contains a definition for\n a function ``f()``, and ``x`` is an instance of ``C``, calling\n ``x.f(1)`` is equivalent to calling ``C.f(x, 1)``.\n\n When a user-defined method object is derived from a class method\n object, the "class instance" stored in ``im_self`` will actually\n be the class itself, so that calling either ``x.f(1)`` or\n ``C.f(1)`` is equivalent to calling ``f(C,1)`` where ``f`` is\n the underlying function.\n\n Note that the transformation from function object to (unbound or\n bound) method object happens each time the attribute is\n retrieved from the class or instance. In some cases, a fruitful\n optimization is to assign the attribute to a local variable and\n call that local variable. Also notice that this transformation\n only happens for user-defined functions; other callable objects\n (and all non-callable objects) are retrieved without\n transformation. It is also important to note that user-defined\n functions which are attributes of a class instance are not\n converted to bound methods; this *only* happens when the\n function is an attribute of the class.\n\n Generator functions\n A function or method which uses the ``yield`` statement (see\n section *The yield statement*) is called a *generator function*.\n Such a function, when called, always returns an iterator object\n which can be used to execute the body of the function: calling\n the iterator\'s ``next()`` method will cause the function to\n execute until it provides a value using the ``yield`` statement.\n When the function executes a ``return`` statement or falls off\n the end, a ``StopIteration`` exception is raised and the\n iterator will have reached the end of the set of values to be\n returned.\n\n Built-in functions\n A built-in function object is a wrapper around a C function.\n Examples of built-in functions are ``len()`` and ``math.sin()``\n (``math`` is a standard built-in module). The number and type of\n the arguments are determined by the C function. Special read-\n only attributes: ``__doc__`` is the function\'s documentation\n string, or ``None`` if unavailable; ``__name__`` is the\n function\'s name; ``__self__`` is set to ``None`` (but see the\n next item); ``__module__`` is the name of the module the\n function was defined in or ``None`` if unavailable.\n\n Built-in methods\n This is really a different disguise of a built-in function, this\n time containing an object passed to the C function as an\n implicit extra argument. An example of a built-in method is\n ``alist.append()``, assuming *alist* is a list object. In this\n case, the special read-only attribute ``__self__`` is set to the\n object denoted by *alist*.\n\n Class Types\n Class types, or "new-style classes," are callable. These\n objects normally act as factories for new instances of\n themselves, but variations are possible for class types that\n override ``__new__()``. The arguments of the call are passed to\n ``__new__()`` and, in the typical case, to ``__init__()`` to\n initialize the new instance.\n\n Classic Classes\n Class objects are described below. When a class object is\n called, a new class instance (also described below) is created\n and returned. This implies a call to the class\'s ``__init__()``\n method if it has one. Any arguments are passed on to the\n ``__init__()`` method. If there is no ``__init__()`` method,\n the class must be called without arguments.\n\n Class instances\n Class instances are described below. Class instances are\n callable only when the class has a ``__call__()`` method;\n ``x(arguments)`` is a shorthand for ``x.__call__(arguments)``.\n\nModules\n Modules are imported by the ``import`` statement (see section *The\n import statement*). A module object has a namespace implemented by\n a dictionary object (this is the dictionary referenced by the\n func_globals attribute of functions defined in the module).\n Attribute references are translated to lookups in this dictionary,\n e.g., ``m.x`` is equivalent to ``m.__dict__["x"]``. A module object\n does not contain the code object used to initialize the module\n (since it isn\'t needed once the initialization is done).\n\n Attribute assignment updates the module\'s namespace dictionary,\n e.g., ``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``.\n\n Special read-only attribute: ``__dict__`` is the module\'s namespace\n as a dictionary object.\n\n **CPython implementation detail:** Because of the way CPython\n clears module dictionaries, the module dictionary will be cleared\n when the module falls out of scope even if the dictionary still has\n live references. To avoid this, copy the dictionary or keep the\n module around while using its dictionary directly.\n\n Predefined (writable) attributes: ``__name__`` is the module\'s\n name; ``__doc__`` is the module\'s documentation string, or ``None``\n if unavailable; ``__file__`` is the pathname of the file from which\n the module was loaded, if it was loaded from a file. The\n ``__file__`` attribute is not present for C modules that are\n statically linked into the interpreter; for extension modules\n loaded dynamically from a shared library, it is the pathname of the\n shared library file.\n\nClasses\n Both class types (new-style classes) and class objects (old-\n style/classic classes) are typically created by class definitions\n (see section *Class definitions*). A class has a namespace\n implemented by a dictionary object. Class attribute references are\n translated to lookups in this dictionary, e.g., ``C.x`` is\n translated to ``C.__dict__["x"]`` (although for new-style classes\n in particular there are a number of hooks which allow for other\n means of locating attributes). When the attribute name is not found\n there, the attribute search continues in the base classes. For\n old-style classes, the search is depth-first, left-to-right in the\n order of occurrence in the base class list. New-style classes use\n the more complex C3 method resolution order which behaves correctly\n even in the presence of \'diamond\' inheritance structures where\n there are multiple inheritance paths leading back to a common\n ancestor. Additional details on the C3 MRO used by new-style\n classes can be found in the documentation accompanying the 2.3\n release at http://www.python.org/download/releases/2.3/mro/.\n\n When a class attribute reference (for class ``C``, say) would yield\n a user-defined function object or an unbound user-defined method\n object whose associated class is either ``C`` or one of its base\n classes, it is transformed into an unbound user-defined method\n object whose ``im_class`` attribute is ``C``. When it would yield a\n class method object, it is transformed into a bound user-defined\n method object whose ``im_self`` attribute is ``C``. When it would\n yield a static method object, it is transformed into the object\n wrapped by the static method object. See section *Implementing\n Descriptors* for another way in which attributes retrieved from a\n class may differ from those actually contained in its ``__dict__``\n (note that only new-style classes support descriptors).\n\n Class attribute assignments update the class\'s dictionary, never\n the dictionary of a base class.\n\n A class object can be called (see above) to yield a class instance\n (see below).\n\n Special attributes: ``__name__`` is the class name; ``__module__``\n is the module name in which the class was defined; ``__dict__`` is\n the dictionary containing the class\'s namespace; ``__bases__`` is a\n tuple (possibly empty or a singleton) containing the base classes,\n in the order of their occurrence in the base class list;\n ``__doc__`` is the class\'s documentation string, or None if\n undefined.\n\nClass instances\n A class instance is created by calling a class object (see above).\n A class instance has a namespace implemented as a dictionary which\n is the first place in which attribute references are searched.\n When an attribute is not found there, and the instance\'s class has\n an attribute by that name, the search continues with the class\n attributes. If a class attribute is found that is a user-defined\n function object or an unbound user-defined method object whose\n associated class is the class (call it ``C``) of the instance for\n which the attribute reference was initiated or one of its bases, it\n is transformed into a bound user-defined method object whose\n ``im_class`` attribute is ``C`` and whose ``im_self`` attribute is\n the instance. Static method and class method objects are also\n transformed, as if they had been retrieved from class ``C``; see\n above under "Classes". See section *Implementing Descriptors* for\n another way in which attributes of a class retrieved via its\n instances may differ from the objects actually stored in the\n class\'s ``__dict__``. If no class attribute is found, and the\n object\'s class has a ``__getattr__()`` method, that is called to\n satisfy the lookup.\n\n Attribute assignments and deletions update the instance\'s\n dictionary, never a class\'s dictionary. If the class has a\n ``__setattr__()`` or ``__delattr__()`` method, this is called\n instead of updating the instance dictionary directly.\n\n Class instances can pretend to be numbers, sequences, or mappings\n if they have methods with certain special names. See section\n *Special method names*.\n\n Special attributes: ``__dict__`` is the attribute dictionary;\n ``__class__`` is the instance\'s class.\n\nFiles\n A file object represents an open file. File objects are created by\n the ``open()`` built-in function, and also by ``os.popen()``,\n ``os.fdopen()``, and the ``makefile()`` method of socket objects\n (and perhaps by other functions or methods provided by extension\n modules). The objects ``sys.stdin``, ``sys.stdout`` and\n ``sys.stderr`` are initialized to file objects corresponding to the\n interpreter\'s standard input, output and error streams. See *File\n Objects* for complete documentation of file objects.\n\nInternal types\n A few types used internally by the interpreter are exposed to the\n user. Their definitions may change with future versions of the\n interpreter, but they are mentioned here for completeness.\n\n Code objects\n Code objects represent *byte-compiled* executable Python code,\n or *bytecode*. The difference between a code object and a\n function object is that the function object contains an explicit\n reference to the function\'s globals (the module in which it was\n defined), while a code object contains no context; also the\n default argument values are stored in the function object, not\n in the code object (because they represent values calculated at\n run-time). Unlike function objects, code objects are immutable\n and contain no references (directly or indirectly) to mutable\n objects.\n\n Special read-only attributes: ``co_name`` gives the function\n name; ``co_argcount`` is the number of positional arguments\n (including arguments with default values); ``co_nlocals`` is the\n number of local variables used by the function (including\n arguments); ``co_varnames`` is a tuple containing the names of\n the local variables (starting with the argument names);\n ``co_cellvars`` is a tuple containing the names of local\n variables that are referenced by nested functions;\n ``co_freevars`` is a tuple containing the names of free\n variables; ``co_code`` is a string representing the sequence of\n bytecode instructions; ``co_consts`` is a tuple containing the\n literals used by the bytecode; ``co_names`` is a tuple\n containing the names used by the bytecode; ``co_filename`` is\n the filename from which the code was compiled;\n ``co_firstlineno`` is the first line number of the function;\n ``co_lnotab`` is a string encoding the mapping from bytecode\n offsets to line numbers (for details see the source code of the\n interpreter); ``co_stacksize`` is the required stack size\n (including local variables); ``co_flags`` is an integer encoding\n a number of flags for the interpreter.\n\n The following flag bits are defined for ``co_flags``: bit\n ``0x04`` is set if the function uses the ``*arguments`` syntax\n to accept an arbitrary number of positional arguments; bit\n ``0x08`` is set if the function uses the ``**keywords`` syntax\n to accept arbitrary keyword arguments; bit ``0x20`` is set if\n the function is a generator.\n\n Future feature declarations (``from __future__ import\n division``) also use bits in ``co_flags`` to indicate whether a\n code object was compiled with a particular feature enabled: bit\n ``0x2000`` is set if the function was compiled with future\n division enabled; bits ``0x10`` and ``0x1000`` were used in\n earlier versions of Python.\n\n Other bits in ``co_flags`` are reserved for internal use.\n\n If a code object represents a function, the first item in\n ``co_consts`` is the documentation string of the function, or\n ``None`` if undefined.\n\n Frame objects\n Frame objects represent execution frames. They may occur in\n traceback objects (see below).\n\n Special read-only attributes: ``f_back`` is to the previous\n stack frame (towards the caller), or ``None`` if this is the\n bottom stack frame; ``f_code`` is the code object being executed\n in this frame; ``f_locals`` is the dictionary used to look up\n local variables; ``f_globals`` is used for global variables;\n ``f_builtins`` is used for built-in (intrinsic) names;\n ``f_restricted`` is a flag indicating whether the function is\n executing in restricted execution mode; ``f_lasti`` gives the\n precise instruction (this is an index into the bytecode string\n of the code object).\n\n Special writable attributes: ``f_trace``, if not ``None``, is a\n function called at the start of each source code line (this is\n used by the debugger); ``f_exc_type``, ``f_exc_value``,\n ``f_exc_traceback`` represent the last exception raised in the\n parent frame provided another exception was ever raised in the\n current frame (in all other cases they are None); ``f_lineno``\n is the current line number of the frame --- writing to this from\n within a trace function jumps to the given line (only for the\n bottom-most frame). A debugger can implement a Jump command\n (aka Set Next Statement) by writing to f_lineno.\n\n Traceback objects\n Traceback objects represent a stack trace of an exception. A\n traceback object is created when an exception occurs. When the\n search for an exception handler unwinds the execution stack, at\n each unwound level a traceback object is inserted in front of\n the current traceback. When an exception handler is entered,\n the stack trace is made available to the program. (See section\n *The try statement*.) It is accessible as ``sys.exc_traceback``,\n and also as the third item of the tuple returned by\n ``sys.exc_info()``. The latter is the preferred interface,\n since it works correctly when the program is using multiple\n threads. When the program contains no suitable handler, the\n stack trace is written (nicely formatted) to the standard error\n stream; if the interpreter is interactive, it is also made\n available to the user as ``sys.last_traceback``.\n\n Special read-only attributes: ``tb_next`` is the next level in\n the stack trace (towards the frame where the exception\n occurred), or ``None`` if there is no next level; ``tb_frame``\n points to the execution frame of the current level;\n ``tb_lineno`` gives the line number where the exception\n occurred; ``tb_lasti`` indicates the precise instruction. The\n line number and last instruction in the traceback may differ\n from the line number of its frame object if the exception\n occurred in a ``try`` statement with no matching except clause\n or with a finally clause.\n\n Slice objects\n Slice objects are used to represent slices when *extended slice\n syntax* is used. This is a slice using two colons, or multiple\n slices or ellipses separated by commas, e.g., ``a[i:j:step]``,\n ``a[i:j, k:l]``, or ``a[..., i:j]``. They are also created by\n the built-in ``slice()`` function.\n\n Special read-only attributes: ``start`` is the lower bound;\n ``stop`` is the upper bound; ``step`` is the step value; each is\n ``None`` if omitted. These attributes can have any type.\n\n Slice objects support one method:\n\n slice.indices(self, length)\n\n This method takes a single integer argument *length* and\n computes information about the extended slice that the slice\n object would describe if applied to a sequence of *length*\n items. It returns a tuple of three integers; respectively\n these are the *start* and *stop* indices and the *step* or\n stride length of the slice. Missing or out-of-bounds indices\n are handled in a manner consistent with regular slices.\n\n New in version 2.3.\n\n Static method objects\n Static method objects provide a way of defeating the\n transformation of function objects to method objects described\n above. A static method object is a wrapper around any other\n object, usually a user-defined method object. When a static\n method object is retrieved from a class or a class instance, the\n object actually returned is the wrapped object, which is not\n subject to any further transformation. Static method objects are\n not themselves callable, although the objects they wrap usually\n are. Static method objects are created by the built-in\n ``staticmethod()`` constructor.\n\n Class method objects\n A class method object, like a static method object, is a wrapper\n around another object that alters the way in which that object\n is retrieved from classes and class instances. The behaviour of\n class method objects upon such retrieval is described above,\n under "User-defined methods". Class method objects are created\n by the built-in ``classmethod()`` constructor.\n', 'typesfunctions': '\nFunctions\n*********\n\nFunction objects are created by function definitions. The only\noperation on a function object is to call it: ``func(argument-list)``.\n\nThere are really two flavors of function objects: built-in functions\nand user-defined functions. Both support the same operation (to call\nthe function), but the implementation is different, hence the\ndifferent object types.\n\nSee *Function definitions* for more information.\n', 'typesmapping': '\nMapping Types --- ``dict``\n**************************\n\nA *mapping* object maps *hashable* values to arbitrary objects.\nMappings are mutable objects. There is currently only one standard\nmapping type, the *dictionary*. (For other containers see the built\nin ``list``, ``set``, and ``tuple`` classes, and the ``collections``\nmodule.)\n\nA dictionary\'s keys are *almost* arbitrary values. Values that are\nnot *hashable*, that is, values containing lists, dictionaries or\nother mutable types (that are compared by value rather than by object\nidentity) may not be used as keys. Numeric types used for keys obey\nthe normal rules for numeric comparison: if two numbers compare equal\n(such as ``1`` and ``1.0``) then they can be used interchangeably to\nindex the same dictionary entry. (Note however, that since computers\nstore floating-point numbers as approximations it is usually unwise to\nuse them as dictionary keys.)\n\nDictionaries can be created by placing a comma-separated list of\n``key: value`` pairs within braces, for example: ``{\'jack\': 4098,\n\'sjoerd\': 4127}`` or ``{4098: \'jack\', 4127: \'sjoerd\'}``, or by the\n``dict`` constructor.\n\nclass class dict(**kwarg)\nclass class dict(mapping, **kwarg)\nclass class dict(iterable, **kwarg)\n\n Return a new dictionary initialized from an optional positional\n argument and a possibly empty set of keyword arguments.\n\n If no positional argument is given, an empty dictionary is created.\n If a positional argument is given and it is a mapping object, a\n dictionary is created with the same key-value pairs as the mapping\n object. Otherwise, the positional argument must be an *iterator*\n object. Each item in the iterable must itself be an iterator with\n exactly two objects. The first object of each item becomes a key\n in the new dictionary, and the second object the corresponding\n value. If a key occurs more than once, the last value for that key\n becomes the corresponding value in the new dictionary.\n\n If keyword arguments are given, the keyword arguments and their\n values are added to the dictionary created from the positional\n argument. If a key being added is already present, the value from\n the keyword argument replaces the value from the positional\n argument.\n\n To illustrate, the following examples all return a dictionary equal\n to ``{"one": 1, "two": 2, "three": 3}``:\n\n >>> a = dict(one=1, two=2, three=3)\n >>> b = {\'one\': 1, \'two\': 2, \'three\': 3}\n >>> c = dict(zip([\'one\', \'two\', \'three\'], [1, 2, 3]))\n >>> d = dict([(\'two\', 2), (\'one\', 1), (\'three\', 3)])\n >>> e = dict({\'three\': 3, \'one\': 1, \'two\': 2})\n >>> a == b == c == d == e\n True\n\n Providing keyword arguments as in the first example only works for\n keys that are valid Python identifiers. Otherwise, any valid keys\n can be used.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for building a dictionary from\n keyword arguments added.\n\n These are the operations that dictionaries support (and therefore,\n custom mapping types should support too):\n\n len(d)\n\n Return the number of items in the dictionary *d*.\n\n d[key]\n\n Return the item of *d* with key *key*. Raises a ``KeyError`` if\n *key* is not in the map.\n\n New in version 2.5: If a subclass of dict defines a method\n ``__missing__()``, if the key *key* is not present, the\n ``d[key]`` operation calls that method with the key *key* as\n argument. The ``d[key]`` operation then returns or raises\n whatever is returned or raised by the ``__missing__(key)`` call\n if the key is not present. No other operations or methods invoke\n ``__missing__()``. If ``__missing__()`` is not defined,\n ``KeyError`` is raised. ``__missing__()`` must be a method; it\n cannot be an instance variable. For an example, see\n ``collections.defaultdict``.\n\n d[key] = value\n\n Set ``d[key]`` to *value*.\n\n del d[key]\n\n Remove ``d[key]`` from *d*. Raises a ``KeyError`` if *key* is\n not in the map.\n\n key in d\n\n Return ``True`` if *d* has a key *key*, else ``False``.\n\n New in version 2.2.\n\n key not in d\n\n Equivalent to ``not key in d``.\n\n New in version 2.2.\n\n iter(d)\n\n Return an iterator over the keys of the dictionary. This is a\n shortcut for ``iterkeys()``.\n\n clear()\n\n Remove all items from the dictionary.\n\n copy()\n\n Return a shallow copy of the dictionary.\n\n fromkeys(seq[, value])\n\n Create a new dictionary with keys from *seq* and values set to\n *value*.\n\n ``fromkeys()`` is a class method that returns a new dictionary.\n *value* defaults to ``None``.\n\n New in version 2.3.\n\n get(key[, default])\n\n Return the value for *key* if *key* is in the dictionary, else\n *default*. If *default* is not given, it defaults to ``None``,\n so that this method never raises a ``KeyError``.\n\n has_key(key)\n\n Test for the presence of *key* in the dictionary. ``has_key()``\n is deprecated in favor of ``key in d``.\n\n items()\n\n Return a copy of the dictionary\'s list of ``(key, value)``\n pairs.\n\n **CPython implementation detail:** Keys and values are listed in\n an arbitrary order which is non-random, varies across Python\n implementations, and depends on the dictionary\'s history of\n insertions and deletions.\n\n If ``items()``, ``keys()``, ``values()``, ``iteritems()``,\n ``iterkeys()``, and ``itervalues()`` are called with no\n intervening modifications to the dictionary, the lists will\n directly correspond. This allows the creation of ``(value,\n key)`` pairs using ``zip()``: ``pairs = zip(d.values(),\n d.keys())``. The same relationship holds for the ``iterkeys()``\n and ``itervalues()`` methods: ``pairs = zip(d.itervalues(),\n d.iterkeys())`` provides the same value for ``pairs``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.iteritems()]``.\n\n iteritems()\n\n Return an iterator over the dictionary\'s ``(key, value)`` pairs.\n See the note for ``dict.items()``.\n\n Using ``iteritems()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n iterkeys()\n\n Return an iterator over the dictionary\'s keys. See the note for\n ``dict.items()``.\n\n Using ``iterkeys()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n itervalues()\n\n Return an iterator over the dictionary\'s values. See the note\n for ``dict.items()``.\n\n Using ``itervalues()`` while adding or deleting entries in the\n dictionary may raise a ``RuntimeError`` or fail to iterate over\n all entries.\n\n New in version 2.2.\n\n keys()\n\n Return a copy of the dictionary\'s list of keys. See the note\n for ``dict.items()``.\n\n pop(key[, default])\n\n If *key* is in the dictionary, remove it and return its value,\n else return *default*. If *default* is not given and *key* is\n not in the dictionary, a ``KeyError`` is raised.\n\n New in version 2.3.\n\n popitem()\n\n Remove and return an arbitrary ``(key, value)`` pair from the\n dictionary.\n\n ``popitem()`` is useful to destructively iterate over a\n dictionary, as often used in set algorithms. If the dictionary\n is empty, calling ``popitem()`` raises a ``KeyError``.\n\n setdefault(key[, default])\n\n If *key* is in the dictionary, return its value. If not, insert\n *key* with a value of *default* and return *default*. *default*\n defaults to ``None``.\n\n update([other])\n\n Update the dictionary with the key/value pairs from *other*,\n overwriting existing keys. Return ``None``.\n\n ``update()`` accepts either another dictionary object or an\n iterable of key/value pairs (as tuples or other iterables of\n length two). If keyword arguments are specified, the dictionary\n is then updated with those key/value pairs: ``d.update(red=1,\n blue=2)``.\n\n Changed in version 2.4: Allowed the argument to be an iterable\n of key/value pairs and allowed keyword arguments.\n\n values()\n\n Return a copy of the dictionary\'s list of values. See the note\n for ``dict.items()``.\n\n viewitems()\n\n Return a new view of the dictionary\'s items (``(key, value)``\n pairs). See below for documentation of view objects.\n\n New in version 2.7.\n\n viewkeys()\n\n Return a new view of the dictionary\'s keys. See below for\n documentation of view objects.\n\n New in version 2.7.\n\n viewvalues()\n\n Return a new view of the dictionary\'s values. See below for\n documentation of view objects.\n\n New in version 2.7.\n\n\nDictionary view objects\n=======================\n\nThe objects returned by ``dict.viewkeys()``, ``dict.viewvalues()`` and\n``dict.viewitems()`` are *view objects*. They provide a dynamic view\non the dictionary\'s entries, which means that when the dictionary\nchanges, the view reflects these changes.\n\nDictionary views can be iterated over to yield their respective data,\nand support membership tests:\n\nlen(dictview)\n\n Return the number of entries in the dictionary.\n\niter(dictview)\n\n Return an iterator over the keys, values or items (represented as\n tuples of ``(key, value)``) in the dictionary.\n\n Keys and values are iterated over in an arbitrary order which is\n non-random, varies across Python implementations, and depends on\n the dictionary\'s history of insertions and deletions. If keys,\n values and items views are iterated over with no intervening\n modifications to the dictionary, the order of items will directly\n correspond. This allows the creation of ``(value, key)`` pairs\n using ``zip()``: ``pairs = zip(d.values(), d.keys())``. Another\n way to create the same list is ``pairs = [(v, k) for (k, v) in\n d.items()]``.\n\n Iterating views while adding or deleting entries in the dictionary\n may raise a ``RuntimeError`` or fail to iterate over all entries.\n\nx in dictview\n\n Return ``True`` if *x* is in the underlying dictionary\'s keys,\n values or items (in the latter case, *x* should be a ``(key,\n value)`` tuple).\n\nKeys views are set-like since their entries are unique and hashable.\nIf all values are hashable, so that (key, value) pairs are unique and\nhashable, then the items view is also set-like. (Values views are not\ntreated as set-like since the entries are generally not unique.) Then\nthese set operations are available ("other" refers either to another\nview or a set):\n\ndictview & other\n\n Return the intersection of the dictview and the other object as a\n new set.\n\ndictview | other\n\n Return the union of the dictview and the other object as a new set.\n\ndictview - other\n\n Return the difference between the dictview and the other object\n (all elements in *dictview* that aren\'t in *other*) as a new set.\n\ndictview ^ other\n\n Return the symmetric difference (all elements either in *dictview*\n or *other*, but not in both) of the dictview and the other object\n as a new set.\n\nAn example of dictionary view usage:\n\n >>> dishes = {\'eggs\': 2, \'sausage\': 1, \'bacon\': 1, \'spam\': 500}\n >>> keys = dishes.viewkeys()\n >>> values = dishes.viewvalues()\n\n >>> # iteration\n >>> n = 0\n >>> for val in values:\n ... n += val\n >>> print(n)\n 504\n\n >>> # keys and values are iterated over in the same order\n >>> list(keys)\n [\'eggs\', \'bacon\', \'sausage\', \'spam\']\n >>> list(values)\n [2, 1, 1, 500]\n\n >>> # view objects are dynamic and reflect dict changes\n >>> del dishes[\'eggs\']\n >>> del dishes[\'sausage\']\n >>> list(keys)\n [\'spam\', \'bacon\']\n\n >>> # set operations\n >>> keys & {\'eggs\', \'bacon\', \'salad\'}\n {\'bacon\'}\n', 'typesmethods': '\nMethods\n*******\n\nMethods are functions that are called using the attribute notation.\nThere are two flavors: built-in methods (such as ``append()`` on\nlists) and class instance methods. Built-in methods are described\nwith the types that support them.\n\nThe implementation adds two special read-only attributes to class\ninstance methods: ``m.im_self`` is the object on which the method\noperates, and ``m.im_func`` is the function implementing the method.\nCalling ``m(arg-1, arg-2, ..., arg-n)`` is completely equivalent to\ncalling ``m.im_func(m.im_self, arg-1, arg-2, ..., arg-n)``.\n\nClass instance methods are either *bound* or *unbound*, referring to\nwhether the method was accessed through an instance or a class,\nrespectively. When a method is unbound, its ``im_self`` attribute\nwill be ``None`` and if called, an explicit ``self`` object must be\npassed as the first argument. In this case, ``self`` must be an\ninstance of the unbound method\'s class (or a subclass of that class),\notherwise a ``TypeError`` is raised.\n\nLike function objects, methods objects support getting arbitrary\nattributes. However, since method attributes are actually stored on\nthe underlying function object (``meth.im_func``), setting method\nattributes on either bound or unbound methods is disallowed.\nAttempting to set an attribute on a method results in an\n``AttributeError`` being raised. In order to set a method attribute,\nyou need to explicitly set it on the underlying function object:\n\n >>> class C:\n ... def method(self):\n ... pass\n ...\n >>> c = C()\n >>> c.method.whoami = \'my name is method\' # can\'t set on the method\n Traceback (most recent call last):\n File "", line 1, in \n AttributeError: \'instancemethod\' object has no attribute \'whoami\'\n >>> c.method.im_func.whoami = \'my name is method\'\n >>> c.method.whoami\n \'my name is method\'\n\nSee *The standard type hierarchy* for more information.\n', 'typesmodules': "\nModules\n*******\n\nThe only special operation on a module is attribute access:\n``m.name``, where *m* is a module and *name* accesses a name defined\nin *m*'s symbol table. Module attributes can be assigned to. (Note\nthat the ``import`` statement is not, strictly speaking, an operation\non a module object; ``import foo`` does not require a module object\nnamed *foo* to exist, rather it requires an (external) *definition*\nfor a module named *foo* somewhere.)\n\nA special attribute of every module is ``__dict__``. This is the\ndictionary containing the module's symbol table. Modifying this\ndictionary will actually change the module's symbol table, but direct\nassignment to the ``__dict__`` attribute is not possible (you can\nwrite ``m.__dict__['a'] = 1``, which defines ``m.a`` to be ``1``, but\nyou can't write ``m.__dict__ = {}``). Modifying ``__dict__`` directly\nis not recommended.\n\nModules built into the interpreter are written like this: ````. If loaded from a file, they are written as\n````.\n", - 'typesseq': '\nSequence Types --- ``str``, ``unicode``, ``list``, ``tuple``, ``bytearray``, ``buffer``, ``xrange``\n***************************************************************************************************\n\nThere are seven sequence types: strings, Unicode strings, lists,\ntuples, bytearrays, buffers, and xrange objects.\n\nFor other containers see the built in ``dict`` and ``set`` classes,\nand the ``collections`` module.\n\nString literals are written in single or double quotes: ``\'xyzzy\'``,\n``"frobozz"``. See *String literals* for more about string literals.\nUnicode strings are much like strings, but are specified in the syntax\nusing a preceding ``\'u\'`` character: ``u\'abc\'``, ``u"def"``. In\naddition to the functionality described here, there are also string-\nspecific methods described in the *String Methods* section. Lists are\nconstructed with square brackets, separating items with commas: ``[a,\nb, c]``. Tuples are constructed by the comma operator (not within\nsquare brackets), with or without enclosing parentheses, but an empty\ntuple must have the enclosing parentheses, such as ``a, b, c`` or\n``()``. A single item tuple must have a trailing comma, such as\n``(d,)``.\n\nBytearray objects are created with the built-in function\n``bytearray()``.\n\nBuffer objects are not directly supported by Python syntax, but can be\ncreated by calling the built-in function ``buffer()``. They don\'t\nsupport concatenation or repetition.\n\nObjects of type xrange are similar to buffers in that there is no\nspecific syntax to create them, but they are created using the\n``xrange()`` function. They don\'t support slicing, concatenation or\nrepetition, and using ``in``, ``not in``, ``min()`` or ``max()`` on\nthem is inefficient.\n\nMost sequence types support the following operations. The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations. The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority). In the table,\n*s* and *t* are sequences of the same type; *n*, *i* and *j* are\nintegers:\n\n+--------------------+----------------------------------+------------+\n| Operation | Result | Notes |\n+====================+==================================+============+\n| ``x in s`` | ``True`` if an item of *s* is | (1) |\n| | equal to *x*, else ``False`` | |\n+--------------------+----------------------------------+------------+\n| ``x not in s`` | ``False`` if an item of *s* is | (1) |\n| | equal to *x*, else ``True`` | |\n+--------------------+----------------------------------+------------+\n| ``s + t`` | the concatenation of *s* and *t* | (6) |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s`` | *n* shallow copies of *s* | (2) |\n| | concatenated | |\n+--------------------+----------------------------------+------------+\n| ``s[i]`` | *i*th item of *s*, origin 0 | (3) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) |\n| | with step *k* | |\n+--------------------+----------------------------------+------------+\n| ``len(s)`` | length of *s* | |\n+--------------------+----------------------------------+------------+\n| ``min(s)`` | smallest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``max(s)`` | largest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.index(i)`` | index of the first occurence of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.count(i)`` | total number of occurences of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons. In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements. This means that to compare equal, every element must compare\nequal and the two sequences must be of the same type and have the same\nlength. (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string or Unicode string object the ``in`` and ``not\n in`` operations act like a substring test. In Python versions\n before 2.3, *x* had to be a string of length 1. In Python 2.3 and\n beyond, *x* may be a string of any length.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n empty sequence of the same type as *s*). Note also that the copies\n are shallow; nested structures are not copied. This often haunts\n new Python programmers; consider:\n\n >>> lists = [[]] * 3\n >>> lists\n [[], [], []]\n >>> lists[0].append(3)\n >>> lists\n [[3], [3], [3]]\n\n What has happened is that ``[[]]`` is a one-element list containing\n an empty list, so all three elements of ``[[]] * 3`` are (pointers\n to) this single empty list. Modifying any of the elements of\n ``lists`` modifies this single list. You can create a list of\n different lists this way:\n\n >>> lists = [[] for i in range(3)]\n >>> lists[0].append(3)\n >>> lists[1].append(5)\n >>> lists[2].append(7)\n >>> lists\n [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n string: ``len(s) + i`` or ``len(s) + j`` is substituted. But note\n that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n items with index *k* such that ``i <= k < j``. If *i* or *j* is\n greater than ``len(s)``, use ``len(s)``. If *i* is omitted or\n ``None``, use ``0``. If *j* is omitted or ``None``, use\n ``len(s)``. If *i* is greater than or equal to *j*, the slice is\n empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n sequence of items with index ``x = i + n*k`` such that ``0 <= n <\n (j-i)/k``. In other words, the indices are ``i``, ``i+k``,\n ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n never including *j*). If *i* or *j* is greater than ``len(s)``,\n use ``len(s)``. If *i* or *j* are omitted or ``None``, they become\n "end" values (which end depends on the sign of *k*). Note, *k*\n cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. **CPython implementation detail:** If *s* and *t* are both strings,\n some Python implementations such as CPython can usually perform an\n in-place optimization for assignments of the form ``s = s + t`` or\n ``s += t``. When applicable, this optimization makes quadratic\n run-time much less likely. This optimization is both version and\n implementation dependent. For performance sensitive code, it is\n preferable to use the ``str.join()`` method which assures\n consistent linear concatenation performance across versions and\n implementations.\n\n Changed in version 2.4: Formerly, string concatenation never\n occurred in-place.\n\n\nString Methods\n==============\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support. Some of them are also available on\n``bytearray`` objects.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbytearray, buffer, xrange* section. To output formatted strings use\ntemplate strings or the ``%`` operator described in the *String\nFormatting Operations* section. Also, see the ``re`` module for string\nfunctions based on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. The column number is reset to zero after each\n newline occurring in the string. If *tabsize* is not given, a tab\n size of ``8`` characters is assumed. This doesn\'t understand other\n non-printing characters or escape sequences.\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters [4] in the string are lowercase\n and there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters [4] in the string are uppercase\n and there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string with all the cased characters [4]\n converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified or ``-1``, then there is\n no limit on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. This method uses the *universal newlines* approach to\n splitting lines. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\n For example, ``\'ab c\\n\\nde fg\\rkl\\r\\n\'.splitlines()`` returns\n ``[\'ab c\', \'\', \'de fg\', \'kl\']``, while the same call with\n ``splitlines(True)`` returns ``[\'ab c\\n\', \'\\n\', \'de fg\\r\',\n \'kl\\r\\n\']``.\n\n Unlike ``split()`` when a delimiter string *sep* is given, this\n method returns an empty list for the empty string, and a terminal\n line break does not result in an extra line.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n ... return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n ... lambda mo: mo.group(0)[0].upper() +\n ... mo.group(0)[1:].lower(),\n ... s)\n ...\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string with all the cased characters [4]\n converted to uppercase. Note that ``str.upper().isupper()`` might\n be ``False`` if ``s`` contains uncased characters or if the Unicode\n category of the resulting character(s) is not "Lu" (Letter,\n uppercase), but e.g. "Lt" (Letter, titlecase).\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than or equal to ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that can be used to form decimal-radix numbers,\n e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\n\nString Formatting Operations\n============================\n\nString and Unicode objects have one unique built-in operation: the\n``%`` operator (modulo). This is also known as the string\n*formatting* or *interpolation* operator. Given ``format % values``\n(where *format* is a string or Unicode object), ``%`` conversion\nspecifications in *format* are replaced with zero or more elements of\n*values*. The effect is similar to the using ``sprintf()`` in the C\nlanguage. If *format* is a Unicode object, or if any of the objects\nbeing converted using the ``%s`` conversion are Unicode objects, the\nresult will also be a Unicode object.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [5] Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n conversion types.\n\n4. Minimum field width (optional). If specified as an ``\'*\'``\n (asterisk), the actual width is read from the next element of the\n tuple in *values*, and the object to convert comes after the\n minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n precision. If specified as ``\'*\'`` (an asterisk), the actual width\n is read from the next element of the tuple in *values*, and the\n value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print \'%(language)s has %(number)03d quote types.\' % \\\n... {"language": "Python", "number": 2}\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag | Meaning |\n+===========+=======================================================================+\n| ``\'#\'`` | The value conversion will use the "alternate form" (where defined |\n| | below). |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'`` | The conversion will be zero padded for numeric values. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'`` | The converted value is left adjusted (overrides the ``\'0\'`` |\n| | conversion if both are given). |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'`` | (a space) A blank should be left before a positive number (or empty |\n| | string) produced by a signed conversion. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'`` | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion |\n| | (overrides a "space" flag). |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion | Meaning | Notes |\n+==============+=======================================================+=========+\n| ``\'d\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'`` | Signed octal value. | (1) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'`` | Obsolete type -- it is identical to ``\'d\'``. | (7) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'`` | Signed hexadecimal (lowercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'`` | Signed hexadecimal (uppercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'`` | Floating point exponential format (lowercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'`` | Floating point exponential format (uppercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'`` | Floating point format. Uses lowercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'`` | Floating point format. Uses uppercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'`` | Single character (accepts integer or single character | |\n| | string). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'`` | String (converts any Python object using ``repr()``). | (5) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'`` | String (converts any Python object using ``str()``). | (6) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'`` | No argument is converted, results in a ``\'%\'`` | |\n| | character in the result. | |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n point, even if no digits follow it.\n\n The precision determines the number of digits after the decimal\n point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n point, and trailing zeroes are not removed as they would otherwise\n be.\n\n The precision determines the number of significant digits before\n and after the decimal point and defaults to 6.\n\n5. The ``%r`` conversion was added in Python 2.0.\n\n The precision determines the maximal number of characters used.\n\n6. If the object or format provided is a ``unicode`` string, the\n resulting string will also be ``unicode``.\n\n The precision determines the maximal number of characters used.\n\n7. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 2.7: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nXRange Type\n===========\n\nThe ``xrange`` type is an immutable sequence which is commonly used\nfor looping. The advantage of the ``xrange`` type is that an\n``xrange`` object will always take the same amount of memory, no\nmatter the size of the range it represents. There are no consistent\nperformance advantages.\n\nXRange objects have very little behavior: they only support indexing,\niteration, and the ``len()`` function.\n\n\nMutable Sequence Types\n======================\n\nList and ``bytearray`` objects support additional operations that\nallow in-place modification of the object. Other mutable sequence\ntypes (when added to the language) should also support these\noperations. Strings and tuples are immutable sequence types: such\nobjects cannot be modified once created. The following operations are\ndefined on mutable sequence types (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn\'t have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don\'t return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n', + 'typesseq': '\nSequence Types --- ``str``, ``unicode``, ``list``, ``tuple``, ``bytearray``, ``buffer``, ``xrange``\n***************************************************************************************************\n\nThere are seven sequence types: strings, Unicode strings, lists,\ntuples, bytearrays, buffers, and xrange objects.\n\nFor other containers see the built in ``dict`` and ``set`` classes,\nand the ``collections`` module.\n\nString literals are written in single or double quotes: ``\'xyzzy\'``,\n``"frobozz"``. See *String literals* for more about string literals.\nUnicode strings are much like strings, but are specified in the syntax\nusing a preceding ``\'u\'`` character: ``u\'abc\'``, ``u"def"``. In\naddition to the functionality described here, there are also string-\nspecific methods described in the *String Methods* section. Lists are\nconstructed with square brackets, separating items with commas: ``[a,\nb, c]``. Tuples are constructed by the comma operator (not within\nsquare brackets), with or without enclosing parentheses, but an empty\ntuple must have the enclosing parentheses, such as ``a, b, c`` or\n``()``. A single item tuple must have a trailing comma, such as\n``(d,)``.\n\nBytearray objects are created with the built-in function\n``bytearray()``.\n\nBuffer objects are not directly supported by Python syntax, but can be\ncreated by calling the built-in function ``buffer()``. They don\'t\nsupport concatenation or repetition.\n\nObjects of type xrange are similar to buffers in that there is no\nspecific syntax to create them, but they are created using the\n``xrange()`` function. They don\'t support slicing, concatenation or\nrepetition, and using ``in``, ``not in``, ``min()`` or ``max()`` on\nthem is inefficient.\n\nMost sequence types support the following operations. The ``in`` and\n``not in`` operations have the same priorities as the comparison\noperations. The ``+`` and ``*`` operations have the same priority as\nthe corresponding numeric operations. [3] Additional methods are\nprovided for *Mutable Sequence Types*.\n\nThis table lists the sequence operations sorted in ascending priority\n(operations in the same box have the same priority). In the table,\n*s* and *t* are sequences of the same type; *n*, *i* and *j* are\nintegers:\n\n+--------------------+----------------------------------+------------+\n| Operation | Result | Notes |\n+====================+==================================+============+\n| ``x in s`` | ``True`` if an item of *s* is | (1) |\n| | equal to *x*, else ``False`` | |\n+--------------------+----------------------------------+------------+\n| ``x not in s`` | ``False`` if an item of *s* is | (1) |\n| | equal to *x*, else ``True`` | |\n+--------------------+----------------------------------+------------+\n| ``s + t`` | the concatenation of *s* and *t* | (6) |\n+--------------------+----------------------------------+------------+\n| ``s * n, n * s`` | *n* shallow copies of *s* | (2) |\n| | concatenated | |\n+--------------------+----------------------------------+------------+\n| ``s[i]`` | *i*th item of *s*, origin 0 | (3) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) |\n+--------------------+----------------------------------+------------+\n| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) |\n| | with step *k* | |\n+--------------------+----------------------------------+------------+\n| ``len(s)`` | length of *s* | |\n+--------------------+----------------------------------+------------+\n| ``min(s)`` | smallest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``max(s)`` | largest item of *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.index(i)`` | index of the first occurence of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n| ``s.count(i)`` | total number of occurences of | |\n| | *i* in *s* | |\n+--------------------+----------------------------------+------------+\n\nSequence types also support comparisons. In particular, tuples and\nlists are compared lexicographically by comparing corresponding\nelements. This means that to compare equal, every element must compare\nequal and the two sequences must be of the same type and have the same\nlength. (For full details see *Comparisons* in the language\nreference.)\n\nNotes:\n\n1. When *s* is a string or Unicode string object the ``in`` and ``not\n in`` operations act like a substring test. In Python versions\n before 2.3, *x* had to be a string of length 1. In Python 2.3 and\n beyond, *x* may be a string of any length.\n\n2. Values of *n* less than ``0`` are treated as ``0`` (which yields an\n empty sequence of the same type as *s*). Note also that the copies\n are shallow; nested structures are not copied. This often haunts\n new Python programmers; consider:\n\n >>> lists = [[]] * 3\n >>> lists\n [[], [], []]\n >>> lists[0].append(3)\n >>> lists\n [[3], [3], [3]]\n\n What has happened is that ``[[]]`` is a one-element list containing\n an empty list, so all three elements of ``[[]] * 3`` are (pointers\n to) this single empty list. Modifying any of the elements of\n ``lists`` modifies this single list. You can create a list of\n different lists this way:\n\n >>> lists = [[] for i in range(3)]\n >>> lists[0].append(3)\n >>> lists[1].append(5)\n >>> lists[2].append(7)\n >>> lists\n [[3], [5], [7]]\n\n3. If *i* or *j* is negative, the index is relative to the end of the\n string: ``len(s) + i`` or ``len(s) + j`` is substituted. But note\n that ``-0`` is still ``0``.\n\n4. The slice of *s* from *i* to *j* is defined as the sequence of\n items with index *k* such that ``i <= k < j``. If *i* or *j* is\n greater than ``len(s)``, use ``len(s)``. If *i* is omitted or\n ``None``, use ``0``. If *j* is omitted or ``None``, use\n ``len(s)``. If *i* is greater than or equal to *j*, the slice is\n empty.\n\n5. The slice of *s* from *i* to *j* with step *k* is defined as the\n sequence of items with index ``x = i + n*k`` such that ``0 <= n <\n (j-i)/k``. In other words, the indices are ``i``, ``i+k``,\n ``i+2*k``, ``i+3*k`` and so on, stopping when *j* is reached (but\n never including *j*). If *i* or *j* is greater than ``len(s)``,\n use ``len(s)``. If *i* or *j* are omitted or ``None``, they become\n "end" values (which end depends on the sign of *k*). Note, *k*\n cannot be zero. If *k* is ``None``, it is treated like ``1``.\n\n6. **CPython implementation detail:** If *s* and *t* are both strings,\n some Python implementations such as CPython can usually perform an\n in-place optimization for assignments of the form ``s = s + t`` or\n ``s += t``. When applicable, this optimization makes quadratic\n run-time much less likely. This optimization is both version and\n implementation dependent. For performance sensitive code, it is\n preferable to use the ``str.join()`` method which assures\n consistent linear concatenation performance across versions and\n implementations.\n\n Changed in version 2.4: Formerly, string concatenation never\n occurred in-place.\n\n\nString Methods\n==============\n\nBelow are listed the string methods which both 8-bit strings and\nUnicode objects support. Some of them are also available on\n``bytearray`` objects.\n\nIn addition, Python\'s strings support the sequence type methods\ndescribed in the *Sequence Types --- str, unicode, list, tuple,\nbytearray, buffer, xrange* section. To output formatted strings use\ntemplate strings or the ``%`` operator described in the *String\nFormatting Operations* section. Also, see the ``re`` module for string\nfunctions based on regular expressions.\n\nstr.capitalize()\n\n Return a copy of the string with its first character capitalized\n and the rest lowercased.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.center(width[, fillchar])\n\n Return centered in a string of length *width*. Padding is done\n using the specified *fillchar* (default is a space).\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.count(sub[, start[, end]])\n\n Return the number of non-overlapping occurrences of substring *sub*\n in the range [*start*, *end*]. Optional arguments *start* and\n *end* are interpreted as in slice notation.\n\nstr.decode([encoding[, errors]])\n\n Decodes the string using the codec registered for *encoding*.\n *encoding* defaults to the default string encoding. *errors* may\n be given to set a different error handling scheme. The default is\n ``\'strict\'``, meaning that encoding errors raise ``UnicodeError``.\n Other possible values are ``\'ignore\'``, ``\'replace\'`` and any other\n name registered via ``codecs.register_error()``, see section *Codec\n Base Classes*.\n\n New in version 2.2.\n\n Changed in version 2.3: Support for other error handling schemes\n added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.encode([encoding[, errors]])\n\n Return an encoded version of the string. Default encoding is the\n current default string encoding. *errors* may be given to set a\n different error handling scheme. The default for *errors* is\n ``\'strict\'``, meaning that encoding errors raise a\n ``UnicodeError``. Other possible values are ``\'ignore\'``,\n ``\'replace\'``, ``\'xmlcharrefreplace\'``, ``\'backslashreplace\'`` and\n any other name registered via ``codecs.register_error()``, see\n section *Codec Base Classes*. For a list of possible encodings, see\n section *Standard Encodings*.\n\n New in version 2.0.\n\n Changed in version 2.3: Support for ``\'xmlcharrefreplace\'`` and\n ``\'backslashreplace\'`` and other error handling schemes added.\n\n Changed in version 2.7: Support for keyword arguments added.\n\nstr.endswith(suffix[, start[, end]])\n\n Return ``True`` if the string ends with the specified *suffix*,\n otherwise return ``False``. *suffix* can also be a tuple of\n suffixes to look for. With optional *start*, test beginning at\n that position. With optional *end*, stop comparing at that\n position.\n\n Changed in version 2.5: Accept tuples as *suffix*.\n\nstr.expandtabs([tabsize])\n\n Return a copy of the string where all tab characters are replaced\n by one or more spaces, depending on the current column and the\n given tab size. Tab positions occur every *tabsize* characters\n (default is 8, giving tab positions at columns 0, 8, 16 and so on).\n To expand the string, the current column is set to zero and the\n string is examined character by character. If the character is a\n tab (``\\t``), one or more space characters are inserted in the\n result until the current column is equal to the next tab position.\n (The tab character itself is not copied.) If the character is a\n newline (``\\n``) or return (``\\r``), it is copied and the current\n column is reset to zero. Any other character is copied unchanged\n and the current column is incremented by one regardless of how the\n character is represented when printed.\n\n >>> \'01\\t012\\t0123\\t01234\'.expandtabs()\n \'01 012 0123 01234\'\n >>> \'01\\t012\\t0123\\t01234\'.expandtabs(4)\n \'01 012 0123 01234\'\n\nstr.find(sub[, start[, end]])\n\n Return the lowest index in the string where substring *sub* is\n found, such that *sub* is contained in the slice ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` if *sub* is not found.\n\n Note: The ``find()`` method should be used only if you need to know the\n position of *sub*. To check if *sub* is a substring or not, use\n the ``in`` operator:\n\n >>> \'Py\' in \'Python\'\n True\n\nstr.format(*args, **kwargs)\n\n Perform a string formatting operation. The string on which this\n method is called can contain literal text or replacement fields\n delimited by braces ``{}``. Each replacement field contains either\n the numeric index of a positional argument, or the name of a\n keyword argument. Returns a copy of the string where each\n replacement field is replaced with the string value of the\n corresponding argument.\n\n >>> "The sum of 1 + 2 is {0}".format(1+2)\n \'The sum of 1 + 2 is 3\'\n\n See *Format String Syntax* for a description of the various\n formatting options that can be specified in format strings.\n\n This method of string formatting is the new standard in Python 3,\n and should be preferred to the ``%`` formatting described in\n *String Formatting Operations* in new code.\n\n New in version 2.6.\n\nstr.index(sub[, start[, end]])\n\n Like ``find()``, but raise ``ValueError`` when the substring is not\n found.\n\nstr.isalnum()\n\n Return true if all characters in the string are alphanumeric and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isalpha()\n\n Return true if all characters in the string are alphabetic and\n there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isdigit()\n\n Return true if all characters in the string are digits and there is\n at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.islower()\n\n Return true if all cased characters [4] in the string are lowercase\n and there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isspace()\n\n Return true if there are only whitespace characters in the string\n and there is at least one character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.istitle()\n\n Return true if the string is a titlecased string and there is at\n least one character, for example uppercase characters may only\n follow uncased characters and lowercase characters only cased ones.\n Return false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.isupper()\n\n Return true if all cased characters [4] in the string are uppercase\n and there is at least one cased character, false otherwise.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.join(iterable)\n\n Return a string which is the concatenation of the strings in the\n *iterable* *iterable*. The separator between elements is the\n string providing this method.\n\nstr.ljust(width[, fillchar])\n\n Return the string left justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.lower()\n\n Return a copy of the string with all the cased characters [4]\n converted to lowercase.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.lstrip([chars])\n\n Return a copy of the string with leading characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a prefix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.lstrip()\n \'spacious \'\n >>> \'www.example.com\'.lstrip(\'cmowz.\')\n \'example.com\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.partition(sep)\n\n Split the string at the first occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing the string itself, followed by\n two empty strings.\n\n New in version 2.5.\n\nstr.replace(old, new[, count])\n\n Return a copy of the string with all occurrences of substring *old*\n replaced by *new*. If the optional argument *count* is given, only\n the first *count* occurrences are replaced.\n\nstr.rfind(sub[, start[, end]])\n\n Return the highest index in the string where substring *sub* is\n found, such that *sub* is contained within ``s[start:end]``.\n Optional arguments *start* and *end* are interpreted as in slice\n notation. Return ``-1`` on failure.\n\nstr.rindex(sub[, start[, end]])\n\n Like ``rfind()`` but raises ``ValueError`` when the substring *sub*\n is not found.\n\nstr.rjust(width[, fillchar])\n\n Return the string right justified in a string of length *width*.\n Padding is done using the specified *fillchar* (default is a\n space). The original string is returned if *width* is less than or\n equal to ``len(s)``.\n\n Changed in version 2.4: Support for the *fillchar* argument.\n\nstr.rpartition(sep)\n\n Split the string at the last occurrence of *sep*, and return a\n 3-tuple containing the part before the separator, the separator\n itself, and the part after the separator. If the separator is not\n found, return a 3-tuple containing two empty strings, followed by\n the string itself.\n\n New in version 2.5.\n\nstr.rsplit([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit* splits\n are done, the *rightmost* ones. If *sep* is not specified or\n ``None``, any whitespace string is a separator. Except for\n splitting from the right, ``rsplit()`` behaves like ``split()``\n which is described in detail below.\n\n New in version 2.4.\n\nstr.rstrip([chars])\n\n Return a copy of the string with trailing characters removed. The\n *chars* argument is a string specifying the set of characters to be\n removed. If omitted or ``None``, the *chars* argument defaults to\n removing whitespace. The *chars* argument is not a suffix; rather,\n all combinations of its values are stripped:\n\n >>> \' spacious \'.rstrip()\n \' spacious\'\n >>> \'mississippi\'.rstrip(\'ipz\')\n \'mississ\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.split([sep[, maxsplit]])\n\n Return a list of the words in the string, using *sep* as the\n delimiter string. If *maxsplit* is given, at most *maxsplit*\n splits are done (thus, the list will have at most ``maxsplit+1``\n elements). If *maxsplit* is not specified or ``-1``, then there is\n no limit on the number of splits (all possible splits are made).\n\n If *sep* is given, consecutive delimiters are not grouped together\n and are deemed to delimit empty strings (for example,\n ``\'1,,2\'.split(\',\')`` returns ``[\'1\', \'\', \'2\']``). The *sep*\n argument may consist of multiple characters (for example,\n ``\'1<>2<>3\'.split(\'<>\')`` returns ``[\'1\', \'2\', \'3\']``). Splitting\n an empty string with a specified separator returns ``[\'\']``.\n\n If *sep* is not specified or is ``None``, a different splitting\n algorithm is applied: runs of consecutive whitespace are regarded\n as a single separator, and the result will contain no empty strings\n at the start or end if the string has leading or trailing\n whitespace. Consequently, splitting an empty string or a string\n consisting of just whitespace with a ``None`` separator returns\n ``[]``.\n\n For example, ``\' 1 2 3 \'.split()`` returns ``[\'1\', \'2\', \'3\']``,\n and ``\' 1 2 3 \'.split(None, 1)`` returns ``[\'1\', \'2 3 \']``.\n\nstr.splitlines([keepends])\n\n Return a list of the lines in the string, breaking at line\n boundaries. This method uses the *universal newlines* approach to\n splitting lines. Line breaks are not included in the resulting list\n unless *keepends* is given and true.\n\n For example, ``\'ab c\\n\\nde fg\\rkl\\r\\n\'.splitlines()`` returns\n ``[\'ab c\', \'\', \'de fg\', \'kl\']``, while the same call with\n ``splitlines(True)`` returns ``[\'ab c\\n\', \'\\n\', \'de fg\\r\',\n \'kl\\r\\n\']``.\n\n Unlike ``split()`` when a delimiter string *sep* is given, this\n method returns an empty list for the empty string, and a terminal\n line break does not result in an extra line.\n\nstr.startswith(prefix[, start[, end]])\n\n Return ``True`` if string starts with the *prefix*, otherwise\n return ``False``. *prefix* can also be a tuple of prefixes to look\n for. With optional *start*, test string beginning at that\n position. With optional *end*, stop comparing string at that\n position.\n\n Changed in version 2.5: Accept tuples as *prefix*.\n\nstr.strip([chars])\n\n Return a copy of the string with the leading and trailing\n characters removed. The *chars* argument is a string specifying the\n set of characters to be removed. If omitted or ``None``, the\n *chars* argument defaults to removing whitespace. The *chars*\n argument is not a prefix or suffix; rather, all combinations of its\n values are stripped:\n\n >>> \' spacious \'.strip()\n \'spacious\'\n >>> \'www.example.com\'.strip(\'cmowz.\')\n \'example\'\n\n Changed in version 2.2.2: Support for the *chars* argument.\n\nstr.swapcase()\n\n Return a copy of the string with uppercase characters converted to\n lowercase and vice versa.\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.title()\n\n Return a titlecased version of the string where words start with an\n uppercase character and the remaining characters are lowercase.\n\n The algorithm uses a simple language-independent definition of a\n word as groups of consecutive letters. The definition works in\n many contexts but it means that apostrophes in contractions and\n possessives form word boundaries, which may not be the desired\n result:\n\n >>> "they\'re bill\'s friends from the UK".title()\n "They\'Re Bill\'S Friends From The Uk"\n\n A workaround for apostrophes can be constructed using regular\n expressions:\n\n >>> import re\n >>> def titlecase(s):\n ... return re.sub(r"[A-Za-z]+(\'[A-Za-z]+)?",\n ... lambda mo: mo.group(0)[0].upper() +\n ... mo.group(0)[1:].lower(),\n ... s)\n ...\n >>> titlecase("they\'re bill\'s friends.")\n "They\'re Bill\'s Friends."\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.translate(table[, deletechars])\n\n Return a copy of the string where all characters occurring in the\n optional argument *deletechars* are removed, and the remaining\n characters have been mapped through the given translation table,\n which must be a string of length 256.\n\n You can use the ``maketrans()`` helper function in the ``string``\n module to create a translation table. For string objects, set the\n *table* argument to ``None`` for translations that only delete\n characters:\n\n >>> \'read this short text\'.translate(None, \'aeiou\')\n \'rd ths shrt txt\'\n\n New in version 2.6: Support for a ``None`` *table* argument.\n\n For Unicode objects, the ``translate()`` method does not accept the\n optional *deletechars* argument. Instead, it returns a copy of the\n *s* where all characters have been mapped through the given\n translation table which must be a mapping of Unicode ordinals to\n Unicode ordinals, Unicode strings or ``None``. Unmapped characters\n are left untouched. Characters mapped to ``None`` are deleted.\n Note, a more flexible approach is to create a custom character\n mapping codec using the ``codecs`` module (see ``encodings.cp1251``\n for an example).\n\nstr.upper()\n\n Return a copy of the string with all the cased characters [4]\n converted to uppercase. Note that ``str.upper().isupper()`` might\n be ``False`` if ``s`` contains uncased characters or if the Unicode\n category of the resulting character(s) is not "Lu" (Letter,\n uppercase), but e.g. "Lt" (Letter, titlecase).\n\n For 8-bit strings, this method is locale-dependent.\n\nstr.zfill(width)\n\n Return the numeric string left filled with zeros in a string of\n length *width*. A sign prefix is handled correctly. The original\n string is returned if *width* is less than or equal to ``len(s)``.\n\n New in version 2.2.2.\n\nThe following methods are present only on unicode objects:\n\nunicode.isnumeric()\n\n Return ``True`` if there are only numeric characters in S,\n ``False`` otherwise. Numeric characters include digit characters,\n and all characters that have the Unicode numeric value property,\n e.g. U+2155, VULGAR FRACTION ONE FIFTH.\n\nunicode.isdecimal()\n\n Return ``True`` if there are only decimal characters in S,\n ``False`` otherwise. Decimal characters include digit characters,\n and all characters that can be used to form decimal-radix numbers,\n e.g. U+0660, ARABIC-INDIC DIGIT ZERO.\n\n\nString Formatting Operations\n============================\n\nString and Unicode objects have one unique built-in operation: the\n``%`` operator (modulo). This is also known as the string\n*formatting* or *interpolation* operator. Given ``format % values``\n(where *format* is a string or Unicode object), ``%`` conversion\nspecifications in *format* are replaced with zero or more elements of\n*values*. The effect is similar to the using ``sprintf()`` in the C\nlanguage. If *format* is a Unicode object, or if any of the objects\nbeing converted using the ``%s`` conversion are Unicode objects, the\nresult will also be a Unicode object.\n\nIf *format* requires a single argument, *values* may be a single non-\ntuple object. [5] Otherwise, *values* must be a tuple with exactly\nthe number of items specified by the format string, or a single\nmapping object (for example, a dictionary).\n\nA conversion specifier contains two or more characters and has the\nfollowing components, which must occur in this order:\n\n1. The ``\'%\'`` character, which marks the start of the specifier.\n\n2. Mapping key (optional), consisting of a parenthesised sequence of\n characters (for example, ``(somename)``).\n\n3. Conversion flags (optional), which affect the result of some\n conversion types.\n\n4. Minimum field width (optional). If specified as an ``\'*\'``\n (asterisk), the actual width is read from the next element of the\n tuple in *values*, and the object to convert comes after the\n minimum field width and optional precision.\n\n5. Precision (optional), given as a ``\'.\'`` (dot) followed by the\n precision. If specified as ``\'*\'`` (an asterisk), the actual width\n is read from the next element of the tuple in *values*, and the\n value to convert comes after the precision.\n\n6. Length modifier (optional).\n\n7. Conversion type.\n\nWhen the right argument is a dictionary (or other mapping type), then\nthe formats in the string *must* include a parenthesised mapping key\ninto that dictionary inserted immediately after the ``\'%\'`` character.\nThe mapping key selects the value to be formatted from the mapping.\nFor example:\n\n>>> print \'%(language)s has %(number)03d quote types.\' % \\\n... {"language": "Python", "number": 2}\nPython has 002 quote types.\n\nIn this case no ``*`` specifiers may occur in a format (since they\nrequire a sequential parameter list).\n\nThe conversion flag characters are:\n\n+-----------+-----------------------------------------------------------------------+\n| Flag | Meaning |\n+===========+=======================================================================+\n| ``\'#\'`` | The value conversion will use the "alternate form" (where defined |\n| | below). |\n+-----------+-----------------------------------------------------------------------+\n| ``\'0\'`` | The conversion will be zero padded for numeric values. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'-\'`` | The converted value is left adjusted (overrides the ``\'0\'`` |\n| | conversion if both are given). |\n+-----------+-----------------------------------------------------------------------+\n| ``\' \'`` | (a space) A blank should be left before a positive number (or empty |\n| | string) produced by a signed conversion. |\n+-----------+-----------------------------------------------------------------------+\n| ``\'+\'`` | A sign character (``\'+\'`` or ``\'-\'``) will precede the conversion |\n| | (overrides a "space" flag). |\n+-----------+-----------------------------------------------------------------------+\n\nA length modifier (``h``, ``l``, or ``L``) may be present, but is\nignored as it is not necessary for Python -- so e.g. ``%ld`` is\nidentical to ``%d``.\n\nThe conversion types are:\n\n+--------------+-------------------------------------------------------+---------+\n| Conversion | Meaning | Notes |\n+==============+=======================================================+=========+\n| ``\'d\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'i\'`` | Signed integer decimal. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'o\'`` | Signed octal value. | (1) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'u\'`` | Obsolete type -- it is identical to ``\'d\'``. | (7) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'x\'`` | Signed hexadecimal (lowercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'X\'`` | Signed hexadecimal (uppercase). | (2) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'e\'`` | Floating point exponential format (lowercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'E\'`` | Floating point exponential format (uppercase). | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'f\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'F\'`` | Floating point decimal format. | (3) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'g\'`` | Floating point format. Uses lowercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'G\'`` | Floating point format. Uses uppercase exponential | (4) |\n| | format if exponent is less than -4 or not less than | |\n| | precision, decimal format otherwise. | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'c\'`` | Single character (accepts integer or single character | |\n| | string). | |\n+--------------+-------------------------------------------------------+---------+\n| ``\'r\'`` | String (converts any Python object using *repr()*). | (5) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'s\'`` | String (converts any Python object using ``str()``). | (6) |\n+--------------+-------------------------------------------------------+---------+\n| ``\'%\'`` | No argument is converted, results in a ``\'%\'`` | |\n| | character in the result. | |\n+--------------+-------------------------------------------------------+---------+\n\nNotes:\n\n1. The alternate form causes a leading zero (``\'0\'``) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n2. The alternate form causes a leading ``\'0x\'`` or ``\'0X\'`` (depending\n on whether the ``\'x\'`` or ``\'X\'`` format was used) to be inserted\n between left-hand padding and the formatting of the number if the\n leading character of the result is not already a zero.\n\n3. The alternate form causes the result to always contain a decimal\n point, even if no digits follow it.\n\n The precision determines the number of digits after the decimal\n point and defaults to 6.\n\n4. The alternate form causes the result to always contain a decimal\n point, and trailing zeroes are not removed as they would otherwise\n be.\n\n The precision determines the number of significant digits before\n and after the decimal point and defaults to 6.\n\n5. The ``%r`` conversion was added in Python 2.0.\n\n The precision determines the maximal number of characters used.\n\n6. If the object or format provided is a ``unicode`` string, the\n resulting string will also be ``unicode``.\n\n The precision determines the maximal number of characters used.\n\n7. See **PEP 237**.\n\nSince Python strings have an explicit length, ``%s`` conversions do\nnot assume that ``\'\\0\'`` is the end of the string.\n\nChanged in version 2.7: ``%f`` conversions for numbers whose absolute\nvalue is over 1e50 are no longer replaced by ``%g`` conversions.\n\nAdditional string operations are defined in standard modules\n``string`` and ``re``.\n\n\nXRange Type\n===========\n\nThe ``xrange`` type is an immutable sequence which is commonly used\nfor looping. The advantage of the ``xrange`` type is that an\n``xrange`` object will always take the same amount of memory, no\nmatter the size of the range it represents. There are no consistent\nperformance advantages.\n\nXRange objects have very little behavior: they only support indexing,\niteration, and the ``len()`` function.\n\n\nMutable Sequence Types\n======================\n\nList and ``bytearray`` objects support additional operations that\nallow in-place modification of the object. Other mutable sequence\ntypes (when added to the language) should also support these\noperations. Strings and tuples are immutable sequence types: such\nobjects cannot be modified once created. The following operations are\ndefined on mutable sequence types (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*\'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn\'t have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don\'t return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n', 'typesseq-mutable': "\nMutable Sequence Types\n**********************\n\nList and ``bytearray`` objects support additional operations that\nallow in-place modification of the object. Other mutable sequence\ntypes (when added to the language) should also support these\noperations. Strings and tuples are immutable sequence types: such\nobjects cannot be modified once created. The following operations are\ndefined on mutable sequence types (where *x* is an arbitrary object):\n\n+--------------------------------+----------------------------------+-----------------------+\n| Operation | Result | Notes |\n+================================+==================================+=======================+\n| ``s[i] = x`` | item *i* of *s* is replaced by | |\n| | *x* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j] = t`` | slice of *s* from *i* to *j* is | |\n| | replaced by the contents of the | |\n| | iterable *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j]`` | same as ``s[i:j] = []`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` are | (1) |\n| | replaced by those of *t* | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``del s[i:j:k]`` | removes the elements of | |\n| | ``s[i:j:k]`` from the list | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.append(x)`` | same as ``s[len(s):len(s)] = | (2) |\n| | [x]`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.extend(x)`` | same as ``s[len(s):len(s)] = x`` | (3) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.count(x)`` | return number of *i*'s for which | |\n| | ``s[i] == x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.index(x[, i[, j]])`` | return smallest *k* such that | (4) |\n| | ``s[k] == x`` and ``i <= k < j`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.insert(i, x)`` | same as ``s[i:i] = [x]`` | (5) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.pop([i])`` | same as ``x = s[i]; del s[i]; | (6) |\n| | return x`` | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.remove(x)`` | same as ``del s[s.index(x)]`` | (4) |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.reverse()`` | reverses the items of *s* in | (7) |\n| | place | |\n+--------------------------------+----------------------------------+-----------------------+\n| ``s.sort([cmp[, key[, | sort the items of *s* in place | (7)(8)(9)(10) |\n| reverse]]])`` | | |\n+--------------------------------+----------------------------------+-----------------------+\n\nNotes:\n\n1. *t* must have the same length as the slice it is replacing.\n\n2. The C implementation of Python has historically accepted multiple\n parameters and implicitly joined them into a tuple; this no longer\n works in Python 2.0. Use of this misfeature has been deprecated\n since Python 1.4.\n\n3. *x* can be any iterable object.\n\n4. Raises ``ValueError`` when *x* is not found in *s*. When a negative\n index is passed as the second or third parameter to the ``index()``\n method, the list length is added, as for slice indices. If it is\n still negative, it is truncated to zero, as for slice indices.\n\n Changed in version 2.3: Previously, ``index()`` didn't have\n arguments for specifying start and stop positions.\n\n5. When a negative index is passed as the first parameter to the\n ``insert()`` method, the list length is added, as for slice\n indices. If it is still negative, it is truncated to zero, as for\n slice indices.\n\n Changed in version 2.3: Previously, all negative indices were\n truncated to zero.\n\n6. The ``pop()`` method is only supported by the list and array types.\n The optional argument *i* defaults to ``-1``, so that by default\n the last item is removed and returned.\n\n7. The ``sort()`` and ``reverse()`` methods modify the list in place\n for economy of space when sorting or reversing a large list. To\n remind you that they operate by side effect, they don't return the\n sorted or reversed list.\n\n8. The ``sort()`` method takes optional arguments for controlling the\n comparisons.\n\n *cmp* specifies a custom comparison function of two arguments (list\n items) which should return a negative, zero or positive number\n depending on whether the first argument is considered smaller than,\n equal to, or larger than the second argument: ``cmp=lambda x,y:\n cmp(x.lower(), y.lower())``. The default value is ``None``.\n\n *key* specifies a function of one argument that is used to extract\n a comparison key from each list element: ``key=str.lower``. The\n default value is ``None``.\n\n *reverse* is a boolean value. If set to ``True``, then the list\n elements are sorted as if each comparison were reversed.\n\n In general, the *key* and *reverse* conversion processes are much\n faster than specifying an equivalent *cmp* function. This is\n because *cmp* is called multiple times for each list element while\n *key* and *reverse* touch each element only once. Use\n ``functools.cmp_to_key()`` to convert an old-style *cmp* function\n to a *key* function.\n\n Changed in version 2.3: Support for ``None`` as an equivalent to\n omitting *cmp* was added.\n\n Changed in version 2.4: Support for *key* and *reverse* was added.\n\n9. Starting with Python 2.3, the ``sort()`` method is guaranteed to be\n stable. A sort is stable if it guarantees not to change the\n relative order of elements that compare equal --- this is helpful\n for sorting in multiple passes (for example, sort by department,\n then by salary grade).\n\n10. **CPython implementation detail:** While a list is being sorted,\n the effect of attempting to mutate, or even inspect, the list is\n undefined. The C implementation of Python 2.3 and newer makes the\n list appear empty for the duration, and raises ``ValueError`` if\n it can detect that the list has been mutated during a sort.\n", 'unary': '\nUnary arithmetic and bitwise operations\n***************************************\n\nAll unary arithmetic and bitwise operations have the same priority:\n\n u_expr ::= power | "-" u_expr | "+" u_expr | "~" u_expr\n\nThe unary ``-`` (minus) operator yields the negation of its numeric\nargument.\n\nThe unary ``+`` (plus) operator yields its numeric argument unchanged.\n\nThe unary ``~`` (invert) operator yields the bitwise inversion of its\nplain or long integer argument. The bitwise inversion of ``x`` is\ndefined as ``-(x+1)``. It only applies to integral numbers.\n\nIn all three cases, if the argument does not have the proper type, a\n``TypeError`` exception is raised.\n', 'while': '\nThe ``while`` statement\n***********************\n\nThe ``while`` statement is used for repeated execution as long as an\nexpression is true:\n\n while_stmt ::= "while" expression ":" suite\n ["else" ":" suite]\n\nThis repeatedly tests the expression and, if it is true, executes the\nfirst suite; if the expression is false (which may be the first time\nit is tested) the suite of the ``else`` clause, if present, is\nexecuted and the loop terminates.\n\nA ``break`` statement executed in the first suite terminates the loop\nwithout executing the ``else`` clause\'s suite. A ``continue``\nstatement executed in the first suite skips the rest of the suite and\ngoes back to testing the expression.\n', 'with': '\nThe ``with`` statement\n**********************\n\nNew in version 2.5.\n\nThe ``with`` statement is used to wrap the execution of a block with\nmethods defined by a context manager (see section *With Statement\nContext Managers*). This allows common\n``try``...``except``...``finally`` usage patterns to be encapsulated\nfor convenient reuse.\n\n with_stmt ::= "with" with_item ("," with_item)* ":" suite\n with_item ::= expression ["as" target]\n\nThe execution of the ``with`` statement with one "item" proceeds as\nfollows:\n\n1. The context expression (the expression given in the ``with_item``)\n is evaluated to obtain a context manager.\n\n2. The context manager\'s ``__exit__()`` is loaded for later use.\n\n3. The context manager\'s ``__enter__()`` method is invoked.\n\n4. If a target was included in the ``with`` statement, the return\n value from ``__enter__()`` is assigned to it.\n\n Note: The ``with`` statement guarantees that if the ``__enter__()``\n method returns without an error, then ``__exit__()`` will always\n be called. Thus, if an error occurs during the assignment to the\n target list, it will be treated the same as an error occurring\n within the suite would be. See step 6 below.\n\n5. The suite is executed.\n\n6. The context manager\'s ``__exit__()`` method is invoked. If an\n exception caused the suite to be exited, its type, value, and\n traceback are passed as arguments to ``__exit__()``. Otherwise,\n three ``None`` arguments are supplied.\n\n If the suite was exited due to an exception, and the return value\n from the ``__exit__()`` method was false, the exception is\n reraised. If the return value was true, the exception is\n suppressed, and execution continues with the statement following\n the ``with`` statement.\n\n If the suite was exited for any reason other than an exception, the\n return value from ``__exit__()`` is ignored, and execution proceeds\n at the normal location for the kind of exit that was taken.\n\nWith more than one item, the context managers are processed as if\nmultiple ``with`` statements were nested:\n\n with A() as a, B() as b:\n suite\n\nis equivalent to\n\n with A() as a:\n with B() as b:\n suite\n\nNote: In Python 2.5, the ``with`` statement is only allowed when the\n ``with_statement`` feature has been enabled. It is always enabled\n in Python 2.6.\n\nChanged in version 2.7: Support for multiple context expressions.\n\nSee also:\n\n **PEP 0343** - The "with" statement\n The specification, background, and examples for the Python\n ``with`` statement.\n', - 'yield': '\nThe ``yield`` statement\n***********************\n\n yield_stmt ::= yield_expression\n\nThe ``yield`` statement is only used when defining a generator\nfunction, and is only used in the body of the generator function.\nUsing a ``yield`` statement in a function definition is sufficient to\ncause that definition to create a generator function instead of a\nnormal function.\n\nWhen a generator function is called, it returns an iterator known as a\ngenerator iterator, or more commonly, a generator. The body of the\ngenerator function is executed by calling the generator\'s ``next()``\nmethod repeatedly until it raises an exception.\n\nWhen a ``yield`` statement is executed, the state of the generator is\nfrozen and the value of ``expression_list`` is returned to\n``next()``\'s caller. By "frozen" we mean that all local state is\nretained, including the current bindings of local variables, the\ninstruction pointer, and the internal evaluation stack: enough\ninformation is saved so that the next time ``next()`` is invoked, the\nfunction can proceed exactly as if the ``yield`` statement were just\nanother external call.\n\nAs of Python version 2.5, the ``yield`` statement is now allowed in\nthe ``try`` clause of a ``try`` ... ``finally`` construct. If the\ngenerator is not resumed before it is finalized (by reaching a zero\nreference count or by being garbage collected), the generator-\niterator\'s ``close()`` method will be called, allowing any pending\n``finally`` clauses to execute.\n\nNote: In Python 2.2, the ``yield`` statement was only allowed when the\n ``generators`` feature has been enabled. This ``__future__`` import\n statement was used to enable the feature:\n\n from __future__ import generators\n\nSee also:\n\n **PEP 0255** - Simple Generators\n The proposal for adding generators and the ``yield`` statement\n to Python.\n\n **PEP 0342** - Coroutines via Enhanced Generators\n The proposal that, among other generator enhancements, proposed\n allowing ``yield`` to appear inside a ``try`` ... ``finally``\n block.\n'} + 'yield': '\nThe ``yield`` statement\n***********************\n\n yield_stmt ::= yield_expression\n\nThe ``yield`` statement is only used when defining a generator\nfunction, and is only used in the body of the generator function.\nUsing a ``yield`` statement in a function definition is sufficient to\ncause that definition to create a generator function instead of a\nnormal function.\n\nWhen a generator function is called, it returns an iterator known as a\ngenerator iterator, or more commonly, a generator. The body of the\ngenerator function is executed by calling the generator\'s ``next()``\nmethod repeatedly until it raises an exception.\n\nWhen a ``yield`` statement is executed, the state of the generator is\nfrozen and the value of ``expression_list`` is returned to\n``next()``\'s caller. By "frozen" we mean that all local state is\nretained, including the current bindings of local variables, the\ninstruction pointer, and the internal evaluation stack: enough\ninformation is saved so that the next time ``next()`` is invoked, the\nfunction can proceed exactly as if the ``yield`` statement were just\nanother external call.\n\nAs of Python version 2.5, the ``yield`` statement is now allowed in\nthe ``try`` clause of a ``try`` ... ``finally`` construct. If the\ngenerator is not resumed before it is finalized (by reaching a zero\nreference count or by being garbage collected), the generator-\niterator\'s ``close()`` method will be called, allowing any pending\n``finally`` clauses to execute.\n\nFor full details of ``yield`` semantics, refer to the *Yield\nexpressions* section.\n\nNote: In Python 2.2, the ``yield`` statement was only allowed when the\n ``generators`` feature has been enabled. This ``__future__`` import\n statement was used to enable the feature:\n\n from __future__ import generators\n\nSee also:\n\n **PEP 0255** - Simple Generators\n The proposal for adding generators and the ``yield`` statement\n to Python.\n\n **PEP 0342** - Coroutines via Enhanced Generators\n The proposal that, among other generator enhancements, proposed\n allowing ``yield`` to appear inside a ``try`` ... ``finally``\n block.\n'} diff --git a/Lib/random.py b/Lib/random.py index af04ab207d3..1a3a13ee862 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -170,29 +170,28 @@ def __reduce__(self): ## -------------------- integer methods ------------------- - def randrange(self, start, stop=None, step=1, int=int, default=None, - maxwidth=1L< 0: - if istart >= maxwidth: + if istart >= _maxwidth: return self._randbelow(istart) - return int(self.random() * istart) + return _int(self.random() * istart) raise ValueError, "empty range for randrange()" # stop argument supplied. - istop = int(stop) + istop = _int(stop) if istop != stop: raise ValueError, "non-integer stop for randrange()" width = istop - istart @@ -210,14 +209,14 @@ def randrange(self, start, stop=None, step=1, int=int, default=None, # a long, but we're supposed to return an int (for backward # compatibility). - if width >= maxwidth: - return int(istart + self._randbelow(width)) - return int(istart + int(self.random()*width)) + if width >= _maxwidth: + return _int(istart + self._randbelow(width)) + return _int(istart + _int(self.random()*width)) if step == 1: raise ValueError, "empty range for randrange() (%d,%d, %d)" % (istart, istop, width) # Non-unit step argument supplied. - istep = int(step) + istep = _int(step) if istep != step: raise ValueError, "non-integer step for randrange()" if istep > 0: @@ -230,9 +229,9 @@ def randrange(self, start, stop=None, step=1, int=int, default=None, if n <= 0: raise ValueError, "empty range for randrange()" - if n >= maxwidth: + if n >= _maxwidth: return istart + istep*self._randbelow(n) - return istart + istep*int(self.random() * n) + return istart + istep*_int(self.random() * n) def randint(self, a, b): """Return random integer in range [a, b], including both end points. @@ -240,7 +239,7 @@ def randint(self, a, b): return self.randrange(a, b+1) - def _randbelow(self, n, _log=_log, int=int, _maxwidth=1L< n-1 > 2**(k-2) + k = _int(1.00001 + _log(n-1, 2.0)) # 2**k > n-1 > 2**(k-2) r = getrandbits(k) while r >= n: r = getrandbits(k) @@ -265,7 +264,7 @@ def _randbelow(self, n, _log=_log, int=int, _maxwidth=1L<= _maxwidth: _warn("Underlying random() generator does not supply \n" "enough bits to choose from a population range this large") - return int(self.random() * n) + return _int(self.random() * n) ## -------------------- sequence methods ------------------- @@ -273,18 +272,20 @@ def choice(self, seq): """Choose a random element from a non-empty sequence.""" return seq[int(self.random() * len(seq))] # raises IndexError if seq is empty - def shuffle(self, x, random=None, int=int): + def shuffle(self, x, random=None): """x, random=random.random -> shuffle list x in place; return None. Optional arg random is a 0-argument function returning a random float in [0.0, 1.0); by default, the standard random.random. + """ if random is None: random = self.random + _int = int for i in reversed(xrange(1, len(x))): # pick an element in x[:i+1] with which to exchange x[i] - j = int(random() * (i+1)) + j = _int(random() * (i+1)) x[i], x[j] = x[j], x[i] def sample(self, population, k): diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index 366c59c0cdf..6e4bd12ad31 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -116,7 +116,7 @@ def attr_matches(self, text): """Compute matches when text contains a dot. Assuming the text is of the form NAME.NAME....[NAME], and is - evaluatable in self.namespace, it will be evaluated and its attributes + evaluable in self.namespace, it will be evaluated and its attributes (as revealed by dir()) are used as possible completions. (For class instances, class members are also considered.) diff --git a/Lib/robotparser.py b/Lib/robotparser.py index 1722863d144..ad3be947115 100644 --- a/Lib/robotparser.py +++ b/Lib/robotparser.py @@ -160,6 +160,7 @@ def __init__(self, path, allowance): if path == '' and not allowance: # an empty value means allow all allowance = True + path = urlparse.urlunparse(urlparse.urlparse(path)) self.path = urllib.quote(path) self.allowance = allowance diff --git a/Lib/site.py b/Lib/site.py index 54e515479a9..f1b0ae8c645 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -114,18 +114,6 @@ def removeduppaths(): sys.path[:] = L return known_paths -# XXX This should not be part of site.py, since it is needed even when -# using the -S option for Python. See http://www.python.org/sf/586680 -def addbuilddir(): - """Append ./build/lib. in case we're running in the build dir - (especially for Guido :-)""" - from sysconfig import get_platform - s = "build/lib.%s-%.3s" % (get_platform(), sys.version) - if hasattr(sys, 'gettotalrefcount'): - s += '-pydebug' - s = os.path.join(os.path.dirname(sys.path.pop()), s) - sys.path.append(s) - def _init_pathinfo(): """Return a set containing all existing directory entries from sys.path""" @@ -537,9 +525,6 @@ def main(): abs__file__() known_paths = removeduppaths() - if (os.name == "posix" and sys.path and - os.path.basename(sys.path[-1]) == "Modules"): - addbuilddir() if ENABLE_USER_SITE is None: ENABLE_USER_SITE = check_enableusersite() known_paths = addusersitepackages(known_paths) diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 9e8bc72379f..7f07840044b 100755 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -237,10 +237,12 @@ def __init__(self, host='', port=0, local_hostname=None, If specified, `host' is the name of the remote host to which to connect. If specified, `port' specifies the port to which to connect. - By default, smtplib.SMTP_PORT is used. An SMTPConnectError is raised - if the specified `host' doesn't respond correctly. If specified, - `local_hostname` is used as the FQDN of the local host. By default, - the local hostname is found using socket.getfqdn(). + By default, smtplib.SMTP_PORT is used. If a host is specified the + connect method is called, and if it returns anything other than a + success code an SMTPConnectError is raised. If specified, + `local_hostname` is used as the FQDN of the local host for the + HELO/EHLO command. Otherwise, the local hostname is found using + socket.getfqdn(). """ self.timeout = timeout @@ -758,12 +760,15 @@ def quit(self): if _have_ssl: class SMTP_SSL(SMTP): - """ This is a subclass derived from SMTP that connects over an SSL encrypted - socket (to use this class you need a socket module that was compiled with SSL - support). If host is not specified, '' (the local host) is used. If port is - omitted, the standard SMTP-over-SSL port (465) is used. keyfile and certfile - are also optional - they can contain a PEM formatted private key and - certificate chain file for the SSL connection. + """ This is a subclass derived from SMTP that connects over an SSL + encrypted socket (to use this class you need a socket module that was + compiled with SSL support). If host is not specified, '' (the local + host) is used. If port is omitted, the standard SMTP-over-SSL port + (465) is used. local_hostname has the same meaning as it does in the + SMTP class. keyfile and certfile are also optional - they can contain + a PEM formatted private key and certificate chain file for the SSL + connection. + """ default_port = SMTP_SSL_PORT @@ -794,9 +799,10 @@ class LMTP(SMTP): """LMTP - Local Mail Transfer Protocol The LMTP protocol, which is very similar to ESMTP, is heavily based - on the standard SMTP client. It's common to use Unix sockets for LMTP, - so our connect() method must support that as well as a regular - host:port server. To specify a Unix socket, you must use an absolute + on the standard SMTP client. It's common to use Unix sockets for + LMTP, so our connect() method must support that as well as a regular + host:port server. local_hostname has the same meaning as it does in + the SMTP class. To specify a Unix socket, you must use an absolute path as the host, starting with a '/'. Authentication is supported, using the regular SMTP mechanism. When diff --git a/Lib/sqlite3/test/regression.py b/Lib/sqlite3/test/regression.py index 8a39d59071b..4c25f0005c8 100644 --- a/Lib/sqlite3/test/regression.py +++ b/Lib/sqlite3/test/regression.py @@ -159,7 +159,8 @@ def CheckSetIsolationLevel(self): def CheckCursorConstructorCallCheck(self): """ - Verifies that cursor methods check wether base class __init__ was called. + Verifies that cursor methods check whether base class __init__ was + called. """ class Cursor(sqlite.Cursor): def __init__(self, con): @@ -177,7 +178,8 @@ def __init__(self, con): def CheckConnectionConstructorCallCheck(self): """ - Verifies that connection methods check wether base class __init__ was called. + Verifies that connection methods check whether base class __init__ was + called. """ class Connection(sqlite.Connection): def __init__(self, name): diff --git a/Lib/sqlite3/test/types.py b/Lib/sqlite3/test/types.py index c5ab39bb771..400a4f24086 100644 --- a/Lib/sqlite3/test/types.py +++ b/Lib/sqlite3/test/types.py @@ -244,7 +244,7 @@ def CheckNumber1(self): self.assertEqual(type(value), float) def CheckNumber2(self): - """Checks wether converter names are cut off at '(' characters""" + """Checks whether converter names are cut off at '(' characters""" self.cur.execute("insert into test(n2) values (5)") value = self.cur.execute("select n2 from test").fetchone()[0] # if the converter is not used, it's an int instead of a float diff --git a/Lib/sre_compile.py b/Lib/sre_compile.py index 7cda2b657b8..97c1663b0b9 100644 --- a/Lib/sre_compile.py +++ b/Lib/sre_compile.py @@ -13,7 +13,6 @@ import _sre, sys import sre_parse from sre_constants import * -from _sre import MAXREPEAT assert _sre.MAGIC == MAGIC, "SRE module mismatch" @@ -355,8 +354,6 @@ def _optimize_unicode(charset, fixup): def _simple(av): # check if av is a "simple" operator lo, hi = av[2].getwidth() - if lo == 0 and hi == MAXREPEAT: - raise error, "nothing to repeat" return lo == hi == 1 and av[2][0][0] != SUBPATTERN def _compile_info(code, pattern, flags): diff --git a/Lib/sre_constants.py b/Lib/sre_constants.py index b0175e71be6..69224e274c5 100644 --- a/Lib/sre_constants.py +++ b/Lib/sre_constants.py @@ -15,7 +15,11 @@ MAGIC = 20031017 -from _sre import MAXREPEAT +try: + from _sre import MAXREPEAT +except ImportError: + import _sre + MAXREPEAT = _sre.MAXREPEAT = 65535 # SRE standard exception (access as sre.error) # should this really be here? diff --git a/Lib/sre_parse.py b/Lib/sre_parse.py index a0cf34414b6..e37e2cfce3f 100644 --- a/Lib/sre_parse.py +++ b/Lib/sre_parse.py @@ -15,7 +15,6 @@ import sys from sre_constants import * -from _sre import MAXREPEAT SPECIAL_CHARS = ".\\[{()*+?^$|" REPEAT_CHARS = "*+?{" @@ -142,12 +141,12 @@ def getwidth(self): # determine the width (min, max) for this subpattern if self.width: return self.width - lo = hi = 0L + lo = hi = 0 UNITCODES = (ANY, RANGE, IN, LITERAL, NOT_LITERAL, CATEGORY) REPEATCODES = (MIN_REPEAT, MAX_REPEAT) for op, av in self.data: if op is BRANCH: - i = sys.maxint + i = MAXREPEAT - 1 j = 0 for av in av[1]: l, h = av.getwidth() @@ -165,14 +164,14 @@ def getwidth(self): hi = hi + j elif op in REPEATCODES: i, j = av[2].getwidth() - lo = lo + long(i) * av[0] - hi = hi + long(j) * av[1] + lo = lo + i * av[0] + hi = hi + j * av[1] elif op in UNITCODES: lo = lo + 1 hi = hi + 1 elif op == SUCCESS: break - self.width = int(min(lo, sys.maxint)), int(min(hi, sys.maxint)) + self.width = min(lo, MAXREPEAT - 1), min(hi, MAXREPEAT) return self.width class Tokenizer: @@ -549,7 +548,8 @@ def _parse(source, state): if not name: raise error("missing group name") if not isname(name): - raise error, "bad character in group name" + raise error("bad character in group name %r" % + name) elif sourcematch("="): # named backreference name = "" @@ -563,7 +563,8 @@ def _parse(source, state): if not name: raise error("missing group name") if not isname(name): - raise error, "bad character in group name" + raise error("bad character in backref group name " + "%r" % name) gid = state.groupdict.get(name) if gid is None: raise error, "unknown group name" diff --git a/Lib/ssl.py b/Lib/ssl.py index 88296358a02..329b9d10d64 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -344,17 +344,21 @@ def accept(self): SSL channel, and the address of the remote client.""" newsock, addr = socket.accept(self) - return (SSLSocket(newsock, - keyfile=self.keyfile, - certfile=self.certfile, - server_side=True, - cert_reqs=self.cert_reqs, - ssl_version=self.ssl_version, - ca_certs=self.ca_certs, - ciphers=self.ciphers, - do_handshake_on_connect=self.do_handshake_on_connect, - suppress_ragged_eofs=self.suppress_ragged_eofs), - addr) + try: + return (SSLSocket(newsock, + keyfile=self.keyfile, + certfile=self.certfile, + server_side=True, + cert_reqs=self.cert_reqs, + ssl_version=self.ssl_version, + ca_certs=self.ca_certs, + ciphers=self.ciphers, + do_handshake_on_connect=self.do_handshake_on_connect, + suppress_ragged_eofs=self.suppress_ragged_eofs), + addr) + except socket_error as e: + newsock.close() + raise e def makefile(self, mode='r', bufsize=-1): diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 309f9a39728..3108537c504 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -2,8 +2,6 @@ # # For more information about this module, see PEP 324. # -# This module should remain compatible with Python 2.2, see PEP 291. -# # Copyright (c) 2003-2005 by Peter Astrand # # Licensed to PSF under a Contributor Agreement. @@ -145,7 +143,7 @@ class Popen(args, bufsize=0, executable=None, started to execute, will be re-raised in the parent. Additionally, the exception object will have one extra attribute called 'child_traceback', which is a string containing traceback information -from the childs point of view. +from the child's point of view. The most common exception raised is OSError. This occurs, for example, when trying to execute a non-existent file. Applications @@ -700,12 +698,12 @@ def __init__(self, args, bufsize=0, executable=None, (p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite) = self._get_handles(stdin, stdout, stderr) + errread, errwrite), to_close = self._get_handles(stdin, stdout, stderr) try: self._execute_child(args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, - startupinfo, creationflags, shell, + startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) @@ -713,18 +711,12 @@ def __init__(self, args, bufsize=0, executable=None, # Preserve original exception in case os.close raises. exc_type, exc_value, exc_trace = sys.exc_info() - to_close = [] - # Only close the pipes we created. - if stdin == PIPE: - to_close.extend((p2cread, p2cwrite)) - if stdout == PIPE: - to_close.extend((c2pread, c2pwrite)) - if stderr == PIPE: - to_close.extend((errread, errwrite)) - for fd in to_close: try: - os.close(fd) + if mswindows: + fd.Close() + else: + os.close(fd) except EnvironmentError: pass @@ -818,8 +810,9 @@ def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ + to_close = set() if stdin is None and stdout is None and stderr is None: - return (None, None, None, None, None, None) + return (None, None, None, None, None, None), to_close p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None @@ -837,6 +830,10 @@ def _get_handles(self, stdin, stdout, stderr): # Assuming file-like object p2cread = msvcrt.get_osfhandle(stdin.fileno()) p2cread = self._make_inheritable(p2cread) + # We just duplicated the handle, it has to be closed at the end + to_close.add(p2cread) + if stdin == PIPE: + to_close.add(p2cwrite) if stdout is None: c2pwrite = _subprocess.GetStdHandle(_subprocess.STD_OUTPUT_HANDLE) @@ -850,6 +847,10 @@ def _get_handles(self, stdin, stdout, stderr): # Assuming file-like object c2pwrite = msvcrt.get_osfhandle(stdout.fileno()) c2pwrite = self._make_inheritable(c2pwrite) + # We just duplicated the handle, it has to be closed at the end + to_close.add(c2pwrite) + if stdout == PIPE: + to_close.add(c2pread) if stderr is None: errwrite = _subprocess.GetStdHandle(_subprocess.STD_ERROR_HANDLE) @@ -865,10 +866,14 @@ def _get_handles(self, stdin, stdout, stderr): # Assuming file-like object errwrite = msvcrt.get_osfhandle(stderr.fileno()) errwrite = self._make_inheritable(errwrite) + # We just duplicated the handle, it has to be closed at the end + to_close.add(errwrite) + if stderr == PIPE: + to_close.add(errread) return (p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite) + errread, errwrite), to_close def _make_inheritable(self, handle): @@ -897,7 +902,7 @@ def _find_w9xpopen(self): def _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, - startupinfo, creationflags, shell, + startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): @@ -936,6 +941,10 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, # kill children. creationflags |= _subprocess.CREATE_NEW_CONSOLE + def _close_in_parent(fd): + fd.Close() + to_close.remove(fd) + # Start the process try: hp, ht, pid, tid = _subprocess.CreateProcess(executable, args, @@ -960,11 +969,11 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, # pipe will not close when the child process exits and the # ReadFile will hang. if p2cread is not None: - p2cread.Close() + _close_in_parent(p2cread) if c2pwrite is not None: - c2pwrite.Close() + _close_in_parent(c2pwrite) if errwrite is not None: - errwrite.Close() + _close_in_parent(errwrite) # Retain the process handle, but close the thread handle self._child_created = True @@ -1090,6 +1099,7 @@ def _get_handles(self, stdin, stdout, stderr): """Construct and return tuple with IO objects: p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite """ + to_close = set() p2cread, p2cwrite = None, None c2pread, c2pwrite = None, None errread, errwrite = None, None @@ -1098,6 +1108,7 @@ def _get_handles(self, stdin, stdout, stderr): pass elif stdin == PIPE: p2cread, p2cwrite = self.pipe_cloexec() + to_close.update((p2cread, p2cwrite)) elif isinstance(stdin, int): p2cread = stdin else: @@ -1108,6 +1119,7 @@ def _get_handles(self, stdin, stdout, stderr): pass elif stdout == PIPE: c2pread, c2pwrite = self.pipe_cloexec() + to_close.update((c2pread, c2pwrite)) elif isinstance(stdout, int): c2pwrite = stdout else: @@ -1118,6 +1130,7 @@ def _get_handles(self, stdin, stdout, stderr): pass elif stderr == PIPE: errread, errwrite = self.pipe_cloexec() + to_close.update((errread, errwrite)) elif stderr == STDOUT: errwrite = c2pwrite elif isinstance(stderr, int): @@ -1128,7 +1141,7 @@ def _get_handles(self, stdin, stdout, stderr): return (p2cread, p2cwrite, c2pread, c2pwrite, - errread, errwrite) + errread, errwrite), to_close def _set_cloexec_flag(self, fd, cloexec=True): @@ -1172,7 +1185,7 @@ def _close_fds(self, but): def _execute_child(self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, - startupinfo, creationflags, shell, + startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): @@ -1191,6 +1204,10 @@ def _execute_child(self, args, executable, preexec_fn, close_fds, if executable is None: executable = args[0] + def _close_in_parent(fd): + os.close(fd) + to_close.remove(fd) + # For transferring possible exec failure from child to parent # The first char specifies the exception type: 0 means # OSError, 1 means some other error. @@ -1249,16 +1266,17 @@ def _dup2(a, b): os.close(fd) closed.add(fd) - # Close all other fds, if asked for - if close_fds: - self._close_fds(but=errpipe_write) - if cwd is not None: os.chdir(cwd) if preexec_fn: preexec_fn() + # Close all other fds, if asked for - after + # preexec_fn(), which may open FDs. + if close_fds: + self._close_fds(but=errpipe_write) + if env is None: os.execvp(executable, args) else: @@ -1284,17 +1302,17 @@ def _dup2(a, b): # be sure the FD is closed no matter what os.close(errpipe_write) - if p2cread is not None and p2cwrite is not None: - os.close(p2cread) - if c2pwrite is not None and c2pread is not None: - os.close(c2pwrite) - if errwrite is not None and errread is not None: - os.close(errwrite) - # Wait for exec to fail or succeed; possibly raising exception # Exception limited to 1M data = _eintr_retry_call(os.read, errpipe_read, 1048576) finally: + if p2cread is not None and p2cwrite is not None: + _close_in_parent(p2cread) + if c2pwrite is not None and c2pread is not None: + _close_in_parent(c2pwrite) + if errwrite is not None and errread is not None: + _close_in_parent(errwrite) + # be sure the FD is closed no matter what os.close(errpipe_read) diff --git a/Lib/sunau.py b/Lib/sunau.py index a04d8c01314..4f0885aaf56 100644 --- a/Lib/sunau.py +++ b/Lib/sunau.py @@ -203,6 +203,10 @@ def initfp(self, file): break else: self._info = '' + try: + self._data_pos = file.tell() + except (AttributeError, IOError): + self._data_pos = None def getfp(self): return self._file @@ -255,7 +259,8 @@ def readframes(self, nframes): if nframes == AUDIO_UNKNOWN_SIZE: data = self._file.read() else: - data = self._file.read(nframes * self._framesize * self._nchannels) + data = self._file.read(nframes * self._framesize) + self._soundpos += len(data) // self._framesize if self._encoding == AUDIO_FILE_ENCODING_MULAW_8: import audioop data = audioop.ulaw2lin(data, self._sampwidth) @@ -263,8 +268,10 @@ def readframes(self, nframes): return None # XXX--not implemented yet def rewind(self): + if self._data_pos is None: + raise IOError('cannot seek') + self._file.seek(self._data_pos) self._soundpos = 0 - self._file.seek(self._hdr_size) def tell(self): return self._soundpos @@ -272,7 +279,9 @@ def tell(self): def setpos(self, pos): if pos < 0 or pos > self.getnframes(): raise Error, 'position not in range' - self._file.seek(pos * self._framesize + self._hdr_size) + if self._data_pos is None: + raise IOError('cannot seek') + self._file.seek(self._data_pos + pos * self._framesize) self._soundpos = pos def close(self): @@ -382,10 +391,10 @@ def tell(self): def writeframesraw(self, data): self._ensure_header_written() - nframes = len(data) / self._framesize if self._comptype == 'ULAW': import audioop data = audioop.lin2ulaw(data, self._sampwidth) + nframes = len(data) // self._framesize self._file.write(data) self._nframeswritten = self._nframeswritten + nframes self._datawritten = self._datawritten + len(data) @@ -445,6 +454,10 @@ def _write_header(self): length = AUDIO_UNKNOWN_SIZE else: length = self._nframes * self._framesize + try: + self._form_length_pos = self._file.tell() + except (AttributeError, IOError): + self._form_length_pos = None _write_u32(self._file, length) self._datalength = length _write_u32(self._file, encoding) @@ -454,7 +467,9 @@ def _write_header(self): self._file.write('\0'*(header_size - len(self._info) - 24)) def _patchheader(self): - self._file.seek(8) + if self._form_length_pos is None: + raise IOError('cannot seek') + self._file.seek(self._form_length_pos) _write_u32(self._file, self._datawritten) self._datalength = self._datawritten self._file.seek(0, 2) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index d74ca391c64..aa69351bf0e 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -278,9 +278,10 @@ def _get_makefile_filename(): return os.path.join(_PROJECT_BASE, "Makefile") return os.path.join(get_path('platstdlib'), "config", "Makefile") - -def _init_posix(vars): - """Initialize the module as appropriate for POSIX systems.""" +def _generate_posix_vars(): + """Generate the Python module containing build-time variables.""" + import pprint + vars = {} # load the installed Makefile: makefile = _get_makefile_filename() try: @@ -308,6 +309,49 @@ def _init_posix(vars): if _PYTHON_BUILD: vars['LDSHARED'] = vars['BLDSHARED'] + # There's a chicken-and-egg situation on OS X with regards to the + # _sysconfigdata module after the changes introduced by #15298: + # get_config_vars() is called by get_platform() as part of the + # `make pybuilddir.txt` target -- which is a precursor to the + # _sysconfigdata.py module being constructed. Unfortunately, + # get_config_vars() eventually calls _init_posix(), which attempts + # to import _sysconfigdata, which we won't have built yet. In order + # for _init_posix() to work, if we're on Darwin, just mock up the + # _sysconfigdata module manually and populate it with the build vars. + # This is more than sufficient for ensuring the subsequent call to + # get_platform() succeeds. + name = '_sysconfigdata' + if 'darwin' in sys.platform: + import imp + module = imp.new_module(name) + module.build_time_vars = vars + sys.modules[name] = module + + pybuilddir = 'build/lib.%s-%s' % (get_platform(), sys.version[:3]) + if hasattr(sys, "gettotalrefcount"): + pybuilddir += '-pydebug' + try: + os.makedirs(pybuilddir) + except OSError: + pass + destfile = os.path.join(pybuilddir, name + '.py') + + with open(destfile, 'wb') as f: + f.write('# system configuration generated and used by' + ' the sysconfig module\n') + f.write('build_time_vars = ') + pprint.pprint(vars, stream=f) + + # Create file used for sys.path fixup -- see Modules/getpath.c + with open('pybuilddir.txt', 'w') as f: + f.write(pybuilddir) + +def _init_posix(vars): + """Initialize the module as appropriate for POSIX systems.""" + # _sysconfigdata is generated at build time, see _generate_posix_vars() + from _sysconfigdata import build_time_vars + vars.update(build_time_vars) + def _init_non_posix(vars): """Initialize the module as appropriate for NT""" # set basic install directories @@ -565,3 +609,28 @@ def get_platform(): def get_python_version(): return _PY_VERSION_SHORT + + +def _print_dict(title, data): + for index, (key, value) in enumerate(sorted(data.items())): + if index == 0: + print '%s: ' % (title) + print '\t%s = "%s"' % (key, value) + + +def _main(): + """Display all information sysconfig detains.""" + if '--generate-posix-vars' in sys.argv: + _generate_posix_vars() + return + print 'Platform: "%s"' % get_platform() + print 'Python version: "%s"' % get_python_version() + print 'Current installation scheme: "%s"' % _get_default_scheme() + print + _print_dict('Paths', get_paths()) + print + _print_dict('Variables', get_config_vars()) + + +if __name__ == '__main__': + _main() diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 89d8cc3ffd5..44ecd24eff1 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -330,7 +330,7 @@ class ExtractError(TarError): """General exception for extract errors.""" pass class ReadError(TarError): - """Exception for unreadble tar archives.""" + """Exception for unreadable tar archives.""" pass class CompressionError(TarError): """Exception for unavailable compression methods.""" @@ -2462,16 +2462,18 @@ def next(self): # Fix for SF #1100429: Under rare circumstances it can # happen that getmembers() is called during iteration, # which will cause TarIter to stop prematurely. - if not self.tarfile._loaded: + + if self.index == 0 and self.tarfile.firstmember is not None: + tarinfo = self.tarfile.next() + elif self.index < len(self.tarfile.members): + tarinfo = self.tarfile.members[self.index] + elif not self.tarfile._loaded: tarinfo = self.tarfile.next() if not tarinfo: self.tarfile._loaded = True raise StopIteration else: - try: - tarinfo = self.tarfile.members[self.index] - except IndexError: - raise StopIteration + raise StopIteration self.index += 1 return tarinfo diff --git a/Lib/tempfile.py b/Lib/tempfile.py index f2ddbb0d9ca..7154d2c6b52 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -242,6 +242,10 @@ def _mkstemp_inner(dir, pre, suf, flags): except OSError, e: if e.errno == _errno.EEXIST: continue # try again + if _os.name == 'nt' and e.errno == _errno.EACCES: + # On windows, when a directory with the chosen name already + # exists, EACCES error code is returned instead of EEXIST. + continue raise raise IOError, (_errno.EEXIST, "No usable temporary file name found") @@ -612,7 +616,7 @@ def writelines(self, iterable): return rv def xreadlines(self, *args): - try: - return self._file.xreadlines(*args) - except AttributeError: + if hasattr(self._file, 'xreadlines'): # real file + return iter(self._file) + else: # StringIO() return iter(self._file.readlines(*args)) diff --git a/Lib/test/inspect_fodder.py b/Lib/test/inspect_fodder.py index afde2e2514b..5c87ae6f828 100644 --- a/Lib/test/inspect_fodder.py +++ b/Lib/test/inspect_fodder.py @@ -49,6 +49,8 @@ def argue(self, a, b, c): class MalodorousPervert(StupidGit): pass +Tit = MalodorousPervert + class ParrotDroppings: pass diff --git a/Lib/test/leakers/test_ctypes.py b/Lib/test/leakers/test_ctypes.py index 0f9a2cdc9a1..7d7e9ff3a11 100644 --- a/Lib/test/leakers/test_ctypes.py +++ b/Lib/test/leakers/test_ctypes.py @@ -1,6 +1,5 @@ # Taken from Lib/ctypes/test/test_keeprefs.py, PointerToStructure.test(). -# When this leak is fixed, remember to remove from Misc/build.sh LEAKY_TESTS. from ctypes import Structure, c_int, POINTER import gc diff --git a/Lib/test/nullbytecert.pem b/Lib/test/nullbytecert.pem new file mode 100644 index 00000000000..447186c9508 --- /dev/null +++ b/Lib/test/nullbytecert.pem @@ -0,0 +1,90 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 0 (0x0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Validity + Not Before: Aug 7 13:11:52 2013 GMT + Not After : Aug 7 13:12:52 2013 GMT + Subject: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b5:ea:ed:c9:fb:46:7d:6f:3b:76:80:dd:3a:f3: + 03:94:0b:a7:a6:db:ec:1d:df:ff:23:74:08:9d:97: + 16:3f:a3:a4:7b:3e:1b:0e:96:59:25:03:a7:26:e2: + 88:a9:cf:79:cd:f7:04:56:b0:ab:79:32:6e:59:c1: + 32:30:54:eb:58:a8:cb:91:f0:42:a5:64:27:cb:d4: + 56:31:88:52:ad:cf:bd:7f:f0:06:64:1f:cc:27:b8: + a3:8b:8c:f3:d8:29:1f:25:0b:f5:46:06:1b:ca:02: + 45:ad:7b:76:0a:9c:bf:bb:b9:ae:0d:16:ab:60:75: + ae:06:3e:9c:7c:31:dc:92:2f:29:1a:e0:4b:0c:91: + 90:6c:e9:37:c5:90:d7:2a:d7:97:15:a3:80:8f:5d: + 7b:49:8f:54:30:d4:97:2c:1c:5b:37:b5:ab:69:30: + 68:43:d3:33:78:4b:02:60:f5:3c:44:80:a1:8f:e7: + f0:0f:d1:5e:87:9e:46:cf:62:fc:f9:bf:0c:65:12: + f1:93:c8:35:79:3f:c8:ec:ec:47:f5:ef:be:44:d5: + ae:82:1e:2d:9a:9f:98:5a:67:65:e1:74:70:7c:cb: + d3:c2:ce:0e:45:49:27:dc:e3:2d:d4:fb:48:0e:2f: + 9e:77:b8:14:46:c0:c4:36:ca:02:ae:6a:91:8c:da: + 2f:85 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 88:5A:55:C0:52:FF:61:CD:52:A3:35:0F:EA:5A:9C:24:38:22:F7:5C + X509v3 Key Usage: + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Subject Alternative Name: + ************************************************************* + WARNING: The values for DNS, email and URI are WRONG. OpenSSL + doesn't print the text after a NULL byte. + ************************************************************* + DNS:altnull.python.org, email:null@python.org, URI:http://null.python.org, IP Address:192.0.2.1, IP Address:2001:DB8:0:0:0:0:0:1 + Signature Algorithm: sha1WithRSAEncryption + ac:4f:45:ef:7d:49:a8:21:70:8e:88:59:3e:d4:36:42:70:f5: + a3:bd:8b:d7:a8:d0:58:f6:31:4a:b1:a4:a6:dd:6f:d9:e8:44: + 3c:b6:0a:71:d6:7f:b1:08:61:9d:60:ce:75:cf:77:0c:d2:37: + 86:02:8d:5e:5d:f9:0f:71:b4:16:a8:c1:3d:23:1c:f1:11:b3: + 56:6e:ca:d0:8d:34:94:e6:87:2a:99:f2:ae:ae:cc:c2:e8:86: + de:08:a8:7f:c5:05:fa:6f:81:a7:82:e6:d0:53:9d:34:f4:ac: + 3e:40:fe:89:57:7a:29:a4:91:7e:0b:c6:51:31:e5:10:2f:a4: + 60:76:cd:95:51:1a:be:8b:a1:b0:fd:ad:52:bd:d7:1b:87:60: + d2:31:c7:17:c4:18:4f:2d:08:25:a3:a7:4f:b7:92:ca:e2:f5: + 25:f1:54:75:81:9d:b3:3d:61:a2:f7:da:ed:e1:c6:6f:2c:60: + 1f:d8:6f:c5:92:05:ab:c9:09:62:49:a9:14:ad:55:11:cc:d6: + 4a:19:94:99:97:37:1d:81:5f:8b:cf:a3:a8:96:44:51:08:3d: + 0b:05:65:12:eb:b6:70:80:88:48:72:4f:c6:c2:da:cf:cd:8e: + 5b:ba:97:2f:60:b4:96:56:49:5e:3a:43:76:63:04:be:2a:f6: + c1:ca:a9:94 +-----BEGIN CERTIFICATE----- +MIIE2DCCA8CgAwIBAgIBADANBgkqhkiG9w0BAQUFADCBxTELMAkGA1UEBhMCVVMx +DzANBgNVBAgMBk9yZWdvbjESMBAGA1UEBwwJQmVhdmVydG9uMSMwIQYDVQQKDBpQ +eXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEgMB4GA1UECwwXUHl0aG9uIENvcmUg +RGV2ZWxvcG1lbnQxJDAiBgNVBAMMG251bGwucHl0aG9uLm9yZwBleGFtcGxlLm9y +ZzEkMCIGCSqGSIb3DQEJARYVcHl0aG9uLWRldkBweXRob24ub3JnMB4XDTEzMDgw +NzEzMTE1MloXDTEzMDgwNzEzMTI1MlowgcUxCzAJBgNVBAYTAlVTMQ8wDQYDVQQI +DAZPcmVnb24xEjAQBgNVBAcMCUJlYXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNv +ZnR3YXJlIEZvdW5kYXRpb24xIDAeBgNVBAsMF1B5dGhvbiBDb3JlIERldmVsb3Bt +ZW50MSQwIgYDVQQDDBtudWxsLnB5dGhvbi5vcmcAZXhhbXBsZS5vcmcxJDAiBgkq +hkiG9w0BCQEWFXB5dGhvbi1kZXZAcHl0aG9uLm9yZzCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALXq7cn7Rn1vO3aA3TrzA5QLp6bb7B3f/yN0CJ2XFj+j +pHs+Gw6WWSUDpybiiKnPec33BFawq3kyblnBMjBU61ioy5HwQqVkJ8vUVjGIUq3P +vX/wBmQfzCe4o4uM89gpHyUL9UYGG8oCRa17dgqcv7u5rg0Wq2B1rgY+nHwx3JIv +KRrgSwyRkGzpN8WQ1yrXlxWjgI9de0mPVDDUlywcWze1q2kwaEPTM3hLAmD1PESA +oY/n8A/RXoeeRs9i/Pm/DGUS8ZPINXk/yOzsR/XvvkTVroIeLZqfmFpnZeF0cHzL +08LODkVJJ9zjLdT7SA4vnne4FEbAxDbKAq5qkYzaL4UCAwEAAaOB0DCBzTAMBgNV +HRMBAf8EAjAAMB0GA1UdDgQWBBSIWlXAUv9hzVKjNQ/qWpwkOCL3XDALBgNVHQ8E +BAMCBeAwgZAGA1UdEQSBiDCBhYIeYWx0bnVsbC5weXRob24ub3JnAGV4YW1wbGUu +Y29tgSBudWxsQHB5dGhvbi5vcmcAdXNlckBleGFtcGxlLm9yZ4YpaHR0cDovL251 +bGwucHl0aG9uLm9yZwBodHRwOi8vZXhhbXBsZS5vcmeHBMAAAgGHECABDbgAAAAA +AAAAAAAAAAEwDQYJKoZIhvcNAQEFBQADggEBAKxPRe99SaghcI6IWT7UNkJw9aO9 +i9eo0Fj2MUqxpKbdb9noRDy2CnHWf7EIYZ1gznXPdwzSN4YCjV5d+Q9xtBaowT0j +HPERs1ZuytCNNJTmhyqZ8q6uzMLoht4IqH/FBfpvgaeC5tBTnTT0rD5A/olXeimk +kX4LxlEx5RAvpGB2zZVRGr6LobD9rVK91xuHYNIxxxfEGE8tCCWjp0+3ksri9SXx +VHWBnbM9YaL32u3hxm8sYB/Yb8WSBavJCWJJqRStVRHM1koZlJmXNx2BX4vPo6iW +RFEIPQsFZRLrtnCAiEhyT8bC2s/Njlu6ly9gtJZWSV46Q3ZjBL4q9sHKqZQ= +-----END CERTIFICATE----- diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 7f43dfb90ba..34cafcb7b86 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -538,6 +538,8 @@ def test_insecure_strings(self): "'abc\"", # open quote and close quote don't match "'abc' ?", # junk after close quote "'\\'", # trailing backslash + "'", # issue #17710 + "' ", # issue #17710 # some tests of the quoting rules #"'abc\"\''", #"'\\\\a\'\'\'\\\'\\\\\''", diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 4f80197f8d7..0a1ca4168a5 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -18,7 +18,7 @@ def to_tuple(t): # These tests are compiled through "exec" -# There should be atleast one test per statement +# There should be at least one test per statement exec_tests = [ # None "None", diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index 134470fb79a..20eceb63505 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -10,7 +10,7 @@ import struct from test import test_support -from test.test_support import TESTFN, run_unittest, unlink +from test.test_support import TESTFN, run_unittest, unlink, HOST from StringIO import StringIO try: @@ -18,7 +18,6 @@ except ImportError: threading = None -HOST = test_support.HOST class dummysocket: def __init__(self): diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index ff2c3700d3b..3f2cee4fded 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -18,6 +18,8 @@ def test_encodestring(self): "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNE" "RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0\nNT" "Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==\n") + # Non-bytes + eq(base64.encodestring(bytearray('abc')), 'YWJj\n') def test_decodestring(self): eq = self.assertEqual @@ -32,6 +34,8 @@ def test_decodestring(self): "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789!@#0^&*();:<>,. []{}") eq(base64.decodestring(''), '') + # Non-bytes + eq(base64.decodestring(bytearray("YWJj\n")), "abc") def test_encode(self): eq = self.assertEqual @@ -73,6 +77,10 @@ def test_b64encode(self): "Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==") # Test with arbitrary alternative characters eq(base64.b64encode('\xd3V\xbeo\xf7\x1d', altchars='*$'), '01a*b$cd') + # Non-bytes + eq(base64.b64encode(bytearray('abcd')), 'YWJjZA==') + self.assertRaises(TypeError, base64.b64encode, + '\xd3V\xbeo\xf7\x1d', altchars=bytearray('*$')) # Test standard alphabet eq(base64.standard_b64encode("www.python.org"), "d3d3LnB5dGhvbi5vcmc=") eq(base64.standard_b64encode("a"), "YQ==") @@ -85,8 +93,12 @@ def test_b64encode(self): "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNE" "RUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0NT" "Y3ODkhQCMwXiYqKCk7Ojw+LC4gW117fQ==") + # Non-bytes + eq(base64.standard_b64encode(bytearray('abcd')), 'YWJjZA==') # Test with 'URL safe' alternative characters eq(base64.urlsafe_b64encode('\xd3V\xbeo\xf7\x1d'), '01a-b_cd') + # Non-bytes + eq(base64.urlsafe_b64encode(bytearray('\xd3V\xbeo\xf7\x1d')), '01a-b_cd') def test_b64decode(self): eq = self.assertEqual @@ -104,6 +116,8 @@ def test_b64decode(self): eq(base64.b64decode(''), '') # Test with arbitrary alternative characters eq(base64.b64decode('01a*b$cd', altchars='*$'), '\xd3V\xbeo\xf7\x1d') + # Non-bytes + eq(base64.b64decode(bytearray("YWJj")), "abc") # Test standard alphabet eq(base64.standard_b64decode("d3d3LnB5dGhvbi5vcmc="), "www.python.org") eq(base64.standard_b64decode("YQ=="), "a") @@ -116,8 +130,12 @@ def test_b64decode(self): "abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789!@#0^&*();:<>,. []{}") + # Non-bytes + eq(base64.standard_b64decode(bytearray("YWJj")), "abc") # Test with 'URL safe' alternative characters eq(base64.urlsafe_b64decode('01a-b_cd'), '\xd3V\xbeo\xf7\x1d') + # Non-bytes + eq(base64.urlsafe_b64decode(bytearray('01a-b_cd')), '\xd3V\xbeo\xf7\x1d') def test_b64decode_error(self): self.assertRaises(TypeError, base64.b64decode, 'abc') @@ -131,6 +149,8 @@ def test_b32encode(self): eq(base64.b32encode('abc'), 'MFRGG===') eq(base64.b32encode('abcd'), 'MFRGGZA=') eq(base64.b32encode('abcde'), 'MFRGGZDF') + # Non-bytes + eq(base64.b32encode(bytearray('abcd')), 'MFRGGZA=') def test_b32decode(self): eq = self.assertEqual @@ -141,6 +161,8 @@ def test_b32decode(self): eq(base64.b32decode('MFRGG==='), 'abc') eq(base64.b32decode('MFRGGZA='), 'abcd') eq(base64.b32decode('MFRGGZDF'), 'abcde') + # Non-bytes + self.assertRaises(TypeError, base64.b32decode, bytearray('MFRGG===')) def test_b32decode_casefold(self): eq = self.assertEqual @@ -171,6 +193,8 @@ def test_b16encode(self): eq = self.assertEqual eq(base64.b16encode('\x01\x02\xab\xcd\xef'), '0102ABCDEF') eq(base64.b16encode('\x00'), '00') + # Non-bytes + eq(base64.b16encode(bytearray('\x01\x02\xab\xcd\xef')), '0102ABCDEF') def test_b16decode(self): eq = self.assertEqual @@ -180,6 +204,8 @@ def test_b16decode(self): self.assertRaises(TypeError, base64.b16decode, '0102abcdef') # Case fold eq(base64.b16decode('0102abcdef', True), '\x01\x02\xab\xcd\xef') + # Non-bytes + eq(base64.b16decode(bytearray("0102ABCDEF")), '\x01\x02\xab\xcd\xef') diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index fdab4e2e6f8..12751146648 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -25,9 +25,6 @@ class BaseTest(unittest.TestCase): DATA_CRLF = 'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' EMPTY_DATA = 'BZh9\x17rE8P\x90\x00\x00\x00\x00' - with open(findfile("testbz2_bigmem.bz2"), "rb") as f: - DATA_BIGMEM = f.read() - if has_cmdline_bunzip2: def decompress(self, data): pop = subprocess.Popen("bunzip2", shell=True, @@ -328,24 +325,6 @@ def testMixedIterationReads(self): self.assertRaises(ValueError, f.readline) self.assertRaises(ValueError, f.readlines) - def test_read_truncated(self): - # Drop the eos_magic field (6 bytes) and CRC (4 bytes). - truncated = self.DATA[:-10] - with open(self.filename, 'wb') as f: - f.write(truncated) - with BZ2File(self.filename) as f: - self.assertRaises(EOFError, f.read) - with BZ2File(self.filename) as f: - self.assertEqual(f.read(len(self.TEXT)), self.TEXT) - self.assertRaises(EOFError, f.read, 1) - # Incomplete 4-byte file header, and block header of at least 146 bits. - for i in range(22): - with open(self.filename, 'wb') as f: - f.write(truncated[:i]) - with BZ2File(self.filename) as f: - self.assertRaises(EOFError, f.read, 1) - - class BZ2CompressorTest(BaseTest): def testCompress(self): # "Test BZ2Compressor.compress()/flush()" @@ -431,9 +410,10 @@ def testBigmem(self, size): # Issue #14398: decompression fails when output data is >=2GB. if size < _4G: self.skipTest("Test needs 5GB of memory to run.") - text = bz2.BZ2Decompressor().decompress(self.DATA_BIGMEM) + compressed = bz2.compress("a" * _4G) + text = bz2.BZ2Decompressor().decompress(compressed) self.assertEqual(len(text), _4G) - self.assertEqual(text.strip("\0"), "") + self.assertEqual(text.strip("a"), "") class FuncTest(BaseTest): @@ -482,9 +462,10 @@ def testDecompressBigmem(self, size): # Issue #14398: decompression fails when output data is >=2GB. if size < _4G: self.skipTest("Test needs 5GB of memory to run.") - text = bz2.decompress(self.DATA_BIGMEM) + compressed = bz2.compress("a" * _4G) + text = bz2.decompress(compressed) self.assertEqual(len(text), _4G) - self.assertEqual(text.strip("\0"), "") + self.assertEqual(text.strip("a"), "") def test_main(): test_support.run_unittest( diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py index f6abe976cf7..7fdd482b942 100644 --- a/Lib/test/test_cgi.py +++ b/Lib/test/test_cgi.py @@ -266,6 +266,29 @@ def test_fieldstorage_multipart(self): got = getattr(fs.list[x], k) self.assertEqual(got, exp) + def test_fieldstorage_multipart_maxline(self): + # Issue #18167 + maxline = 1 << 16 + self.maxDiff = None + def check(content): + data = """ +---123 +Content-Disposition: form-data; name="upload"; filename="fake.txt" +Content-Type: text/plain + +%s +---123-- +""".replace('\n', '\r\n') % content + environ = { + 'CONTENT_LENGTH': str(len(data)), + 'CONTENT_TYPE': 'multipart/form-data; boundary=-123', + 'REQUEST_METHOD': 'POST', + } + self.assertEqual(gen_result(data, environ), {'upload': content}) + check('x' * (maxline - 1)) + check('x' * (maxline - 1) + '\r') + check('x' * (maxline - 1) + '\r' + 'y' * (maxline - 1)) + _qs_result = { 'key1': 'value1', 'key2': ['value2x', 'value2y'], diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 61c2df20c43..ecaf9970a1d 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -66,15 +66,34 @@ def test_xmlcharrefreplace(self): # replace unencodable characters which numeric character entities. # For ascii, latin-1 and charmaps this is completely implemented # in C and should be reasonably fast. - s = u"\u30b9\u30d1\u30e2 \xe4nd eggs" + s = u"\u30b9\u30d1\u30e2 \xe4nd egg\u0161" self.assertEqual( s.encode("ascii", "xmlcharrefreplace"), - "スパモ änd eggs" + "スパモ änd eggš" ) self.assertEqual( s.encode("latin-1", "xmlcharrefreplace"), - "スパモ \xe4nd eggs" + "スパモ \xe4nd eggš" ) + self.assertEqual( + s.encode("iso-8859-15", "xmlcharrefreplace"), + "スパモ \xe4nd egg\xa8" + ) + + def test_xmlcharrefreplace_with_surrogates(self): + tests = [(u'\U0001f49d', '💝'), + (u'\ud83d', '�'), + (u'\udc9d', '�'), + (u'\ud83d\udc9d', '💝' if len(u'\U0001f49d') > 1 else + '��'), + ] + for encoding in ['ascii', 'latin1', 'iso-8859-15']: + for s, exp in tests: + self.assertEqual(s.encode(encoding, 'xmlcharrefreplace'), + exp, msg='%r.encode(%r)' % (s, encoding)) + self.assertEqual((s+'X').encode(encoding, 'xmlcharrefreplace'), + exp+'X', + msg='%r.encode(%r)' % (s + 'X', encoding)) def test_xmlcharnamereplace(self): # This time use a named character entity for unencodable diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 313f81ff656..3aaecb2a6f9 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -17,6 +17,43 @@ TestNT = namedtuple('TestNT', 'x y z') # type used for pickle tests +py273_named_tuple_pickle = '''\ +ccopy_reg +_reconstructor +p0 +(ctest.test_collections +TestNT +p1 +c__builtin__ +tuple +p2 +(I10 +I20 +I30 +tp3 +tp4 +Rp5 +ccollections +OrderedDict +p6 +((lp7 +(lp8 +S'x' +p9 +aI10 +aa(lp10 +S'y' +p11 +aI20 +aa(lp12 +S'z' +p13 +aI30 +aatp14 +Rp15 +b. +''' + class TestNamedTuple(unittest.TestCase): def test_factory(self): @@ -215,6 +252,11 @@ def test_name_conflicts(self): # test __getnewargs__ self.assertEqual(t.__getnewargs__(), values) + def test_pickling_bug_18015(self): + # http://bugs.python.org/issue18015 + pt = pickle.loads(py273_named_tuple_pickle) + self.assertEqual(pt.x, 10) + class ABCTestCase(unittest.TestCase): def validate_abstract_methods(self, abc, *names): diff --git a/Lib/test/test_cookielib.py b/Lib/test/test_cookielib.py index dd0ad325d96..a3c31f67a38 100644 --- a/Lib/test/test_cookielib.py +++ b/Lib/test/test_cookielib.py @@ -329,7 +329,7 @@ class CookieTests(TestCase): ## commas and equals are commonly appear in the cookie value). This also ## means that if you fold multiple Set-Cookie header fields into one, ## comma-separated list, it'll be a headache to parse (at least my head -## starts hurting everytime I think of that code). +## starts hurting every time I think of that code). ## - Expires: You'll get all sorts of date formats in the expires, ## including emtpy expires attributes ("expires="). Be as flexible as you ## can, and certainly don't expect the weekday to be there; if you can't diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 53ca5ab12f1..3f826651292 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -914,7 +914,7 @@ class TestSniffer(unittest.TestCase): 'Tommy''s Place':'Blue Island':'IL':'12/28/02':'Blue Sunday/White Crow' 'Stonecutters ''Seafood'' and Chop House':'Lemont':'IL':'12/19/02':'Week Back' """ - header = '''\ + header1 = '''\ "venue","city","state","date","performers" ''' sample3 = '''\ @@ -933,10 +933,35 @@ class TestSniffer(unittest.TestCase): sample6 = "a|b|c\r\nd|e|f\r\n" sample7 = "'a'|'b'|'c'\r\n'd'|e|f\r\n" +# Issue 18155: Use a delimiter that is a special char to regex: + + header2 = '''\ +"venue"+"city"+"state"+"date"+"performers" +''' + sample8 = """\ +Harry's+ Arlington Heights+ IL+ 2/1/03+ Kimi Hayes +Shark City+ Glendale Heights+ IL+ 12/28/02+ Prezence +Tommy's Place+ Blue Island+ IL+ 12/28/02+ Blue Sunday/White Crow +Stonecutters Seafood and Chop House+ Lemont+ IL+ 12/19/02+ Week Back +""" + sample9 = """\ +'Harry''s'+ Arlington Heights'+ 'IL'+ '2/1/03'+ 'Kimi Hayes' +'Shark City'+ Glendale Heights'+' IL'+ '12/28/02'+ 'Prezence' +'Tommy''s Place'+ Blue Island'+ 'IL'+ '12/28/02'+ 'Blue Sunday/White Crow' +'Stonecutters ''Seafood'' and Chop House'+ 'Lemont'+ 'IL'+ '12/19/02'+ 'Week Back' +""" + def test_has_header(self): sniffer = csv.Sniffer() self.assertEqual(sniffer.has_header(self.sample1), False) - self.assertEqual(sniffer.has_header(self.header+self.sample1), True) + self.assertEqual(sniffer.has_header(self.header1 + self.sample1), + True) + + def test_has_header_regex_special_delimiter(self): + sniffer = csv.Sniffer() + self.assertEqual(sniffer.has_header(self.sample8), False) + self.assertEqual(sniffer.has_header(self.header2 + self.sample8), + True) def test_sniff(self): sniffer = csv.Sniffer() @@ -970,13 +995,24 @@ def test_delimiters(self): dialect = sniffer.sniff(self.sample7) self.assertEqual(dialect.delimiter, "|") self.assertEqual(dialect.quotechar, "'") + dialect = sniffer.sniff(self.sample8) + self.assertEqual(dialect.delimiter, '+') + dialect = sniffer.sniff(self.sample9) + self.assertEqual(dialect.delimiter, '+') + self.assertEqual(dialect.quotechar, "'") def test_doublequote(self): sniffer = csv.Sniffer() - dialect = sniffer.sniff(self.header) + dialect = sniffer.sniff(self.header1) + self.assertFalse(dialect.doublequote) + dialect = sniffer.sniff(self.header2) self.assertFalse(dialect.doublequote) dialect = sniffer.sniff(self.sample2) self.assertTrue(dialect.doublequote) + dialect = sniffer.sniff(self.sample8) + self.assertFalse(dialect.doublequote) + dialect = sniffer.sniff(self.sample9) + self.assertTrue(dialect.doublequote) if not hasattr(sys, "gettotalrefcount"): if test_support.verbose: print "*** skipping leakage tests ***" diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index fa0d469e475..fcf9618c5bc 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -250,6 +250,26 @@ def test_userptr_without_set(stdscr): except curses.panel.error: pass +def test_userptr_memory_leak(stdscr): + w = curses.newwin(10, 10) + p = curses.panel.new_panel(w) + obj = object() + nrefs = sys.getrefcount(obj) + for i in range(100): + p.set_userptr(obj) + + p.set_userptr(None) + if sys.getrefcount(obj) != nrefs: + raise RuntimeError, "set_userptr leaked references" + +def test_userptr_segfault(stdscr): + panel = curses.panel.new_panel(stdscr) + class A: + def __del__(self): + panel.set_userptr(None) + panel.set_userptr(A()) + panel.set_userptr(None) + def test_resize_term(stdscr): if hasattr(curses, 'resizeterm'): lines, cols = curses.LINES, curses.COLS @@ -268,6 +288,8 @@ def main(stdscr): module_funcs(stdscr) window_funcs(stdscr) test_userptr_without_set(stdscr) + test_userptr_memory_leak(stdscr) + test_userptr_segfault(stdscr) test_resize_term(stdscr) test_issue6243(stdscr) finally: diff --git a/Lib/test/test_datetime.py b/Lib/test/test_datetime.py index d246d60797e..13ff5a38a18 100644 --- a/Lib/test/test_datetime.py +++ b/Lib/test/test_datetime.py @@ -124,7 +124,7 @@ def test_pickling_subclass(self): self.assertEqual(derived.tzname(None), 'cookie') ############################################################################# -# Base clase for testing a particular aspect of timedelta, time, date and +# Base class for testing a particular aspect of timedelta, time, date and # datetime comparisons. class HarmlessMixedComparison: diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index bfb3552f55a..07664ac208d 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -4136,6 +4136,20 @@ class C(object): C.__name__ = 'D.E' self.assertEqual((C.__module__, C.__name__), (mod, 'D.E')) + def test_evil_type_name(self): + # A badly placed Py_DECREF in type_set_name led to arbitrary code + # execution while the type structure was not in a sane state, and a + # possible segmentation fault as a result. See bug #16447. + class Nasty(str): + def __del__(self): + C.__name__ = "other" + + class C(object): + pass + + C.__name__ = Nasty("abc") + C.__name__ = "normal" + def test_subclass_right_op(self): # Testing correct dispatch of subclass overloading __r__... diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 0840f59799b..a5685b98eb7 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -33,9 +33,11 @@ def test_keys(self): self.assertEqual(d.keys(), []) d = {'a': 1, 'b': 2} k = d.keys() + self.assertEqual(set(k), {'a', 'b'}) + self.assertIn('a', k) + self.assertIn('b', k) self.assertTrue(d.has_key('a')) self.assertTrue(d.has_key('b')) - self.assertRaises(TypeError, d.keys, None) def test_values(self): diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index f90367660c5..30cfb93be7d 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -112,6 +112,13 @@ def test_keys_set_operations(self): self.assertEqual(d1.viewkeys() ^ set(d3.viewkeys()), {'a', 'b', 'd', 'e'}) + self.assertEqual(d1.viewkeys() - d1.viewkeys(), set()) + self.assertEqual(d1.viewkeys() - d2.viewkeys(), {'a'}) + self.assertEqual(d1.viewkeys() - d3.viewkeys(), {'a', 'b'}) + self.assertEqual(d1.viewkeys() - set(d1.viewkeys()), set()) + self.assertEqual(d1.viewkeys() - set(d2.viewkeys()), {'a'}) + self.assertEqual(d1.viewkeys() - set(d3.viewkeys()), {'a', 'b'}) + def test_items_set_operations(self): d1 = {'a': 1, 'b': 2} d2 = {'a': 2, 'b': 2} @@ -144,6 +151,19 @@ def test_items_set_operations(self): self.assertEqual(d1.viewitems() ^ d3.viewitems(), {('a', 1), ('b', 2), ('d', 4), ('e', 5)}) + self.assertEqual(d1.viewitems() - d1.viewitems(), set()) + self.assertEqual(d1.viewitems() - d2.viewitems(), {('a', 1)}) + self.assertEqual(d1.viewitems() - d3.viewitems(), {('a', 1), ('b', 2)}) + self.assertEqual(d1.viewitems() - set(d1.viewitems()), set()) + self.assertEqual(d1.viewitems() - set(d2.viewitems()), {('a', 1)}) + self.assertEqual(d1.viewitems() - set(d3.viewitems()), + {('a', 1), ('b', 2)}) + + def test_recursive_repr(self): + d = {} + d[42] = d.viewvalues() + self.assertRaises(RuntimeError, repr, d) + diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index 7a35f61396d..d7e409cefc6 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -11,7 +11,7 @@ from test.test_support import (verbose, TESTFN, unlink, run_unittest, import_module) -# Skip test if no fnctl module. +# Skip test if no fcntl module. fcntl = import_module('fcntl') diff --git a/Lib/test/test_file.py b/Lib/test/test_file.py index dffa4b56eb6..ac2aafa1ce4 100644 --- a/Lib/test/test_file.py +++ b/Lib/test/test_file.py @@ -154,16 +154,6 @@ def testModeStrings(self): f.close() self.fail('%r is an invalid file mode' % mode) - def testStdin(self): - # This causes the interpreter to exit on OSF1 v5.1. - if sys.platform != 'osf1V5': - self.assertRaises((IOError, ValueError), sys.stdin.seek, -1) - else: - print(( - ' Skipping sys.stdin.seek(-1), it may crash the interpreter.' - ' Test manually.'), file=sys.__stdout__) - self.assertRaises((IOError, ValueError), sys.stdin.truncate) - def testBadModeArgument(self): # verify that we get a sensible error message for bad mode argument bad_mode = "qwerty" diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index b74cec24f79..e7955cce3ee 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -305,7 +305,7 @@ def testAbles(self): finally: os.unlink(TESTFN) - def testModeStrings(self): + def testInvalidModeStrings(self): # check invalid mode strings for mode in ("", "aU", "wU+", "rw", "rt"): try: @@ -316,6 +316,21 @@ def testModeStrings(self): f.close() self.fail('%r is an invalid file mode' % mode) + def testModeStrings(self): + # test that the mode attribute is correct for various mode strings + # given as init args + try: + for modes in [('w', 'wb'), ('wb', 'wb'), ('wb+', 'rb+'), + ('w+b', 'rb+'), ('a', 'ab'), ('ab', 'ab'), + ('ab+', 'ab+'), ('a+b', 'ab+'), ('r', 'rb'), + ('rb', 'rb'), ('rb+', 'rb+'), ('r+b', 'rb+')]: + # read modes are last so that TESTFN will exist first + with _FileIO(TESTFN, modes[0]) as f: + self.assertEqual(f.mode, modes[1]) + finally: + if os.path.exists(TESTFN): + os.unlink(TESTFN) + def testUnicodeOpen(self): # verify repr works for unicode too f = _FileIO(str(TESTFN), "w") diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index dd30efabb6c..0ad8b02d7b5 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -302,6 +302,23 @@ def __oct__(self): def test_main(): test_support.run_unittest(FormatTest) + def test_precision(self): + INT_MAX = 2147483647 + + f = 1.2 + self.assertEqual(format(f, ".0f"), "1") + self.assertEqual(format(f, ".3f"), "1.200") + with self.assertRaises(ValueError) as cm: + format(f, ".%sf" % (INT_MAX + 1)) + self.assertEqual(str(cm.exception), "precision too big") + + c = complex(f) + self.assertEqual(format(f, ".0f"), "1") + self.assertEqual(format(f, ".3f"), "1.200") + with self.assertRaises(ValueError) as cm: + format(f, ".%sf" % (INT_MAX + 1)) + self.assertEqual(str(cm.exception), "precision too big") + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index c82e8a69bf3..e6aaca5376b 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -17,7 +17,7 @@ from unittest import TestCase from test import test_support -from test.test_support import HOST +from test.test_support import HOST, HOSTv6 threading = test_support.import_module('threading') @@ -474,6 +474,14 @@ def test_mkd(self): def test_rmd(self): self.client.rmd('foo') + def test_cwd(self): + dir = self.client.cwd('/foo') + self.assertEqual(dir, '250 cwd ok') + + def test_mkd(self): + dir = self.client.mkd('/foo') + self.assertEqual(dir, '/foo') + def test_pwd(self): dir = self.client.pwd() self.assertEqual(dir, 'pwd ok') @@ -554,7 +562,7 @@ def test_makepasv(self): class TestIPv6Environment(TestCase): def setUp(self): - self.server = DummyFTPServer((HOST, 0), af=socket.AF_INET6) + self.server = DummyFTPServer((HOSTv6, 0), af=socket.AF_INET6) self.server.start() self.client = ftplib.FTP() self.client.connect(self.server.host, self.server.port) @@ -705,7 +713,7 @@ def testTimeoutDefault(self): self.assertTrue(socket.getdefaulttimeout() is None) socket.setdefaulttimeout(30) try: - ftp = ftplib.FTP("localhost") + ftp = ftplib.FTP(HOST) finally: socket.setdefaulttimeout(None) self.assertEqual(ftp.sock.gettimeout(), 30) @@ -717,7 +725,7 @@ def testTimeoutNone(self): self.assertTrue(socket.getdefaulttimeout() is None) socket.setdefaulttimeout(30) try: - ftp = ftplib.FTP("localhost", timeout=None) + ftp = ftplib.FTP(HOST, timeout=None) finally: socket.setdefaulttimeout(None) self.assertTrue(ftp.sock.gettimeout() is None) diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 9d8ff8ed8a1..d1e656704bd 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -53,13 +53,13 @@ def run_gdb(*args, **env_vars): if not gdbpy_version: raise unittest.SkipTest("gdb not built with embedded python support") -# Verify that "gdb" can load our custom hooks. In theory this should never -# fail, but we don't handle the case of the hooks file not existing if the -# tests are run from an installed Python (we'll produce failures in that case). +# Verify that "gdb" can load our custom hooks, as OS security settings may +# disallow this without a customised .gdbinit. cmd = ['--args', sys.executable] _, gdbpy_errors = run_gdb('--args', sys.executable) if "auto-loading has been declined" in gdbpy_errors: msg = "gdb security settings prevent use of custom hooks: " + raise unittest.SkipTest(msg + gdbpy_errors.rstrip()) def python_is_optimized(): cflags = sysconfig.get_config_vars()['PY_CFLAGS'] @@ -142,30 +142,32 @@ def get_stack_trace(self, source=None, script=None, # Use "args" to invoke gdb, capturing stdout, stderr: out, err = run_gdb(*args, PYTHONHASHSEED='0') - # Ignore some noise on stderr due to the pending breakpoint: - err = err.replace('Function "%s" not defined.\n' % breakpoint, '') - # Ignore some other noise on stderr (http://bugs.python.org/issue8600) - err = err.replace("warning: Unable to find libthread_db matching" - " inferior's thread library, thread debugging will" - " not be available.\n", - '') - err = err.replace("warning: Cannot initialize thread debugging" - " library: Debugger service failed\n", - '') - err = err.replace('warning: Could not load shared library symbols for ' - 'linux-vdso.so.1.\n' - 'Do you need "set solib-search-path" or ' - '"set sysroot"?\n', - '') - err = err.replace('warning: Could not load shared library symbols for ' - 'linux-gate.so.1.\n' - 'Do you need "set solib-search-path" or ' - '"set sysroot"?\n', - '') + errlines = err.splitlines() + unexpected_errlines = [] + + # Ignore some benign messages on stderr. + ignore_patterns = ( + 'Function "%s" not defined.' % breakpoint, + "warning: no loadable sections found in added symbol-file" + " system-supplied DSO", + "warning: Unable to find libthread_db matching" + " inferior's thread library, thread debugging will" + " not be available.", + "warning: Cannot initialize thread debugging" + " library: Debugger service failed", + 'warning: Could not load shared library symbols for ' + 'linux-vdso.so', + 'warning: Could not load shared library symbols for ' + 'linux-gate.so', + 'Do you need "set solib-search-path" or ' + '"set sysroot"?', + ) + for line in errlines: + if not line.startswith(ignore_patterns): + unexpected_errlines.append(line) # Ensure no unexpected error messages: - self.assertEqual(err, '') - + self.assertEqual(unexpected_errlines, []) return out def get_gdb_repr(self, source, diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index b0dc24e504e..a82c56e0e6d 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -289,23 +289,13 @@ def test_fileobj_from_fdopen(self): with gzip.GzipFile(fileobj=f, mode="w") as g: self.assertEqual(g.name, "") - def test_read_truncated(self): - data = data1*50 - buf = io.BytesIO() - with gzip.GzipFile(fileobj=buf, mode="w") as f: - f.write(data) - # Drop the CRC (4 bytes) and file size (4 bytes). - truncated = buf.getvalue()[:-8] - with gzip.GzipFile(fileobj=io.BytesIO(truncated)) as f: - self.assertRaises(EOFError, f.read) - with gzip.GzipFile(fileobj=io.BytesIO(truncated)) as f: - self.assertEqual(f.read(len(data)), data) - self.assertRaises(EOFError, f.read, 1) - # Incomplete 10-byte header. - for i in range(2, 10): - with gzip.GzipFile(fileobj=io.BytesIO(truncated[:i])) as f: - self.assertRaises(EOFError, f.read, 1) - + def test_read_with_extra(self): + # Gzip data with an extra field + gzdata = (b'\x1f\x8b\x08\x04\xb2\x17cQ\x02\xff' + b'\x05\x00Extra' + b'\x0bI-.\x01\x002\xd1Mx\x04\x00\x00\x00') + with gzip.GzipFile(fileobj=io.BytesIO(gzdata)) as f: + self.assertEqual(f.read(), b'Test') def test_main(verbose=None): test_support.run_unittest(TestGzip) diff --git a/Lib/test/test_heapq.py b/Lib/test/test_heapq.py index 73b88f09b9b..c4de593bb82 100644 --- a/Lib/test/test_heapq.py +++ b/Lib/test/test_heapq.py @@ -158,6 +158,15 @@ def test_merge(self): self.assertEqual(sorted(chain(*inputs)), list(self.module.merge(*inputs))) self.assertEqual(list(self.module.merge()), []) + def test_merge_does_not_suppress_index_error(self): + # Issue 19018: Heapq.merge suppresses IndexError from user generator + def iterable(): + s = list(range(10)) + for i in range(20): + yield s[i] # IndexError when i > 10 + with self.assertRaises(IndexError): + list(self.module.merge(iterable(), iterable())) + def test_merge_stability(self): class Int(int): pass diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 5dcedc0fe8b..3f1436004d5 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -313,6 +313,9 @@ def test_get(self): #constructs the path relative to the root directory of the HTTPServer response = self.request(self.tempdir_name + '/test') self.check_status_and_reason(response, 200, data=self.data) + # check for trailing "/" which should return 404. See Issue17324 + response = self.request(self.tempdir_name + '/test/') + self.check_status_and_reason(response, 404) response = self.request(self.tempdir_name + '/') self.check_status_and_reason(response, 200) response = self.request(self.tempdir_name) diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py new file mode 100644 index 00000000000..8552e6fbe44 --- /dev/null +++ b/Lib/test/test_idle.py @@ -0,0 +1,31 @@ +import unittest +from test import test_support as support +from test.test_support import import_module, use_resources + +# Skip test if _thread or _tkinter wasn't built or idlelib was deleted. +import_module('threading') # imported by idlelib.PyShell, imports _thread +tk = import_module('Tkinter') # imports _tkinter +idletest = import_module('idlelib.idle_test') + +# If buildbot improperly sets gui resource (#18365, #18441), remove it +# so requires('gui') tests are skipped while non-gui tests still run. +# If there is a problem with Macs, see #18441, msg 193805 +if use_resources and 'gui' in use_resources: + try: + root = tk.Tk() + root.destroy() + except tk.TclError: + while 'gui' in use_resources: + use_resources.remove('gui') + +# Without test_main present, regrtest.runtest_inner (line1219) calls +# unittest.TestLoader().loadTestsFromModule(this_module) which calls +# load_tests() if it finds it. (Unittest.main does the same.) +load_tests = idletest.load_tests + +if __name__ == '__main__': + # Until unittest supports resources, we emulate regrtest's -ugui + # so loaded tests run the same as if textually present here. + # If any Idle test ever needs another resource, add it to the list. + support.use_resources = ['gui'] # use_resources is initially None + unittest.main(verbosity=2, exit=False) diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 04dcfe992a4..4130cd0b502 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -220,8 +220,23 @@ def test_getclasses(self): [('FesteringGob', mod.FesteringGob), ('MalodorousPervert', mod.MalodorousPervert), ('ParrotDroppings', mod.ParrotDroppings), - ('StupidGit', mod.StupidGit)]) - tree = inspect.getclasstree([cls[1] for cls in classes], 1) + ('StupidGit', mod.StupidGit), + ('Tit', mod.MalodorousPervert), + ]) + tree = inspect.getclasstree([cls[1] for cls in classes]) + self.assertEqual(tree, + [(mod.ParrotDroppings, ()), + [(mod.FesteringGob, (mod.MalodorousPervert, + mod.ParrotDroppings)) + ], + (mod.StupidGit, ()), + [(mod.MalodorousPervert, (mod.StupidGit,)), + [(mod.FesteringGob, (mod.MalodorousPervert, + mod.ParrotDroppings)) + ] + ] + ]) + tree = inspect.getclasstree([cls[1] for cls in classes], True) self.assertEqual(tree, [(mod.ParrotDroppings, ()), (mod.StupidGit, ()), diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 5cd28f5bea5..63a05089313 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -2880,7 +2880,7 @@ def _read(): # The buffered IO layer must check for pending signal # handlers, which in this case will invoke alarm_interrupt(). self.assertRaises(ZeroDivisionError, - wio.write, item * (3 * 1000 * 1000)) + wio.write, item * (support.PIPE_MAX_SIZE // len(item) + 1)) t.join() # We got one byte, get another one and check that it isn't a # repeat of the first one. @@ -2978,7 +2978,7 @@ def check_interrupted_write_retry(self, item, **fdopen_kwargs): select = support.import_module("select") # A quantity that exceeds the buffer size of an anonymous pipe's # write end. - N = 1024 * 1024 + N = support.PIPE_MAX_SIZE r, w = os.pipe() fdopen_kwargs["closefd"] = False # We need a separate thread to read from the pipe and allow the diff --git a/Lib/test/test_kqueue.py b/Lib/test/test_kqueue.py index 2f8ff2f382d..3dffc73ce6a 100644 --- a/Lib/test/test_kqueue.py +++ b/Lib/test/test_kqueue.py @@ -70,13 +70,13 @@ def test_create_event(self): self.assertEqual(ev, ev) self.assertNotEqual(ev, other) - bignum = sys.maxsize * 2 + 1 - ev = select.kevent(bignum, 1, 2, 3, sys.maxsize, bignum) + bignum = 0x7fff + ev = select.kevent(bignum, 1, 2, 3, bignum - 1, bignum) self.assertEqual(ev.ident, bignum) self.assertEqual(ev.filter, 1) self.assertEqual(ev.flags, 2) self.assertEqual(ev.fflags, 3) - self.assertEqual(ev.data, sys.maxsize) + self.assertEqual(ev.data, bignum - 1) self.assertEqual(ev.udata, bignum) self.assertEqual(ev, ev) self.assertNotEqual(ev, other) diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index 3508b560568..788d7a8360a 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -21,6 +21,8 @@ def test_default_data(self): eq(self.db.guess_type("foo.tgz"), ("application/x-tar", "gzip")) eq(self.db.guess_type("foo.tar.gz"), ("application/x-tar", "gzip")) eq(self.db.guess_type("foo.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) + eq(self.db.guess_type("foo.tar.xz"), ("application/x-tar", "xz")) def test_data_urls(self): eq = self.assertEqual diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index f84b20aa441..9fd218418ce 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -470,14 +470,10 @@ def test_empty_file (self): f = open (TESTFN, 'w+b') f.close() with open(TESTFN, "rb") as f : - try: - m = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) - m.close() - self.fail("should not have been able to mmap empty file") - except ValueError as e: - self.assertEqual(e.message, "cannot mmap an empty file") - except: - self.fail("unexpected exception: " + str(e)) + self.assertRaisesRegexp(ValueError, + "cannot mmap an empty file", + mmap.mmap, f.fileno(), 0, + access=mmap.ACCESS_READ) def test_offset (self): f = open (TESTFN, 'w+b') diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index bdf4e622490..579229abca5 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -1385,7 +1385,7 @@ def test_remote(self): authkey = os.urandom(32) manager = QueueManager( - address=('localhost', 0), authkey=authkey, serializer=SERIALIZER + address=(test.test_support.HOST, 0), authkey=authkey, serializer=SERIALIZER ) manager.start() @@ -1423,7 +1423,7 @@ def _putter(cls, address, authkey): def test_rapid_restart(self): authkey = os.urandom(32) manager = QueueManager( - address=('localhost', 0), authkey=authkey, serializer=SERIALIZER) + address=(test.test_support.HOST, 0), authkey=authkey, serializer=SERIALIZER) srvr = manager.get_server() addr = srvr.address # Close the connection.Listener socket which gets opened as a part @@ -2287,15 +2287,15 @@ def test_pool_initializer(self): # Verifies os.close(sys.stdin.fileno) vs. sys.stdin.close() behavior # -def _ThisSubProcess(q): +def _this_sub_process(q): try: item = q.get(block=False) except Queue.Empty: pass -def _TestProcess(q): +def _test_process(q): queue = multiprocessing.Queue() - subProc = multiprocessing.Process(target=_ThisSubProcess, args=(queue,)) + subProc = multiprocessing.Process(target=_this_sub_process, args=(queue,)) subProc.daemon = True subProc.start() subProc.join() @@ -2332,7 +2332,7 @@ class TestStdinBadfiledescriptor(unittest.TestCase): def test_queue_in_process(self): queue = multiprocessing.Queue() - proc = multiprocessing.Process(target=_TestProcess, args=(queue,)) + proc = multiprocessing.Process(target=_test_process, args=(queue,)) proc.start() proc.join() @@ -2430,13 +2430,111 @@ def test_flags(self): [sys.executable, '-E', '-B', '-O', '-c', prog]) child_flags, grandchild_flags = json.loads(data.decode('ascii')) self.assertEqual(child_flags, grandchild_flags) + +# +# Issue #17555: ForkAwareThreadLock +# + +class TestForkAwareThreadLock(unittest.TestCase): + # We recurisvely start processes. Issue #17555 meant that the + # after fork registry would get duplicate entries for the same + # lock. The size of the registry at generation n was ~2**n. + + @classmethod + def child(cls, n, conn): + if n > 1: + p = multiprocessing.Process(target=cls.child, args=(n-1, conn)) + p.start() + p.join() + else: + conn.send(len(util._afterfork_registry)) + conn.close() + + def test_lock(self): + r, w = multiprocessing.Pipe(False) + l = util.ForkAwareThreadLock() + old_size = len(util._afterfork_registry) + p = multiprocessing.Process(target=self.child, args=(5, w)) + p.start() + new_size = r.recv() + p.join() + self.assertLessEqual(new_size, old_size) + +# +# Issue #17097: EINTR should be ignored by recv(), send(), accept() etc +# + +class TestIgnoreEINTR(unittest.TestCase): + + @classmethod + def _test_ignore(cls, conn): + def handler(signum, frame): + pass + signal.signal(signal.SIGUSR1, handler) + conn.send('ready') + x = conn.recv() + conn.send(x) + conn.send_bytes(b'x'*(1024*1024)) # sending 1 MB should block + + @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') + def test_ignore(self): + conn, child_conn = multiprocessing.Pipe() + try: + p = multiprocessing.Process(target=self._test_ignore, + args=(child_conn,)) + p.daemon = True + p.start() + child_conn.close() + self.assertEqual(conn.recv(), 'ready') + time.sleep(0.1) + os.kill(p.pid, signal.SIGUSR1) + time.sleep(0.1) + conn.send(1234) + self.assertEqual(conn.recv(), 1234) + time.sleep(0.1) + os.kill(p.pid, signal.SIGUSR1) + self.assertEqual(conn.recv_bytes(), b'x'*(1024*1024)) + time.sleep(0.1) + p.join() + finally: + conn.close() + + @classmethod + def _test_ignore_listener(cls, conn): + def handler(signum, frame): + pass + signal.signal(signal.SIGUSR1, handler) + l = multiprocessing.connection.Listener() + conn.send(l.address) + a = l.accept() + a.send('welcome') + + @unittest.skipUnless(hasattr(signal, 'SIGUSR1'), 'requires SIGUSR1') + def test_ignore_listener(self): + conn, child_conn = multiprocessing.Pipe() + try: + p = multiprocessing.Process(target=self._test_ignore_listener, + args=(child_conn,)) + p.daemon = True + p.start() + child_conn.close() + address = conn.recv() + time.sleep(0.1) + os.kill(p.pid, signal.SIGUSR1) + time.sleep(0.1) + client = multiprocessing.connection.Client(address) + self.assertEqual(client.recv(), 'welcome') + p.join() + finally: + conn.close() + # # # testcases_other = [OtherTest, TestInvalidHandle, TestInitializers, TestStdinBadfiledescriptor, TestTimeouts, TestNoForkBomb, - TestFlags] + TestFlags, TestForkAwareThreadLock, TestIgnoreEINTR] # # diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index 27954563263..4156c535eff 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -5,9 +5,6 @@ class NetrcTestCase(unittest.TestCase): - def tearDown(self): - os.unlink(temp_filename) - def make_nrc(self, test_data): test_data = textwrap.dedent(test_data) mode = 'w' @@ -15,6 +12,7 @@ def make_nrc(self, test_data): mode += 't' with open(temp_filename, mode) as fp: fp.write(test_data) + self.addCleanup(os.unlink, temp_filename) return netrc.netrc(temp_filename) def test_default(self): @@ -103,6 +101,28 @@ def test_comment_at_end_of_machine_line_pass_has_hash(self): """, '#pass') + @unittest.skipUnless(os.name == 'posix', 'POSIX only test') + def test_security(self): + # This test is incomplete since we are normally not run as root and + # therefore can't test the file ownership being wrong. + d = test_support.TESTFN + os.mkdir(d) + self.addCleanup(test_support.rmtree, d) + fn = os.path.join(d, '.netrc') + with open(fn, 'wt') as f: + f.write("""\ + machine foo.domain.com login bar password pass + default login foo password pass + """) + with test_support.EnvironmentVarGuard() as environ: + environ.set('HOME', d) + os.chmod(fn, 0600) + nrc = netrc.netrc() + self.assertEqual(nrc.hosts['foo.domain.com'], + ('bar', None, 'pass')) + os.chmod(fn, 0o622) + self.assertRaises(netrc.NetrcParseError, netrc.netrc) + def test_main(): test_support.run_unittest(NetrcTestCase) diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py new file mode 100644 index 00000000000..31d41e621cd --- /dev/null +++ b/Lib/test/test_nntplib.py @@ -0,0 +1,65 @@ +import socket +import threading +import nntplib +import time + +from unittest import TestCase +from test import test_support + +HOST = test_support.HOST + + +def server(evt, serv, evil=False): + serv.listen(5) + try: + conn, addr = serv.accept() + except socket.timeout: + pass + else: + if evil: + conn.send("1 I'm too long response" * 3000 + "\n") + else: + conn.send("1 I'm OK response\n") + conn.close() + finally: + serv.close() + evt.set() + + +class BaseServerTest(TestCase): + def setUp(self): + self.evt = threading.Event() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock.settimeout(3) + self.port = test_support.bind_port(self.sock) + threading.Thread( + target=server, + args=(self.evt, self.sock, self.evil)).start() + time.sleep(.1) + + def tearDown(self): + self.evt.wait() + + +class ServerTests(BaseServerTest): + evil = False + + def test_basic_connect(self): + nntp = nntplib.NNTP('localhost', self.port) + nntp.sock.close() + + +class EvilServerTests(BaseServerTest): + evil = True + + def test_too_long_line(self): + self.assertRaises(nntplib.NNTPDataError, + nntplib.NNTP, 'localhost', self.port) + + +def test_main(verbose=None): + test_support.run_unittest(EvilServerTests) + test_support.run_unittest(ServerTests) + +if __name__ == '__main__': + test_main() diff --git a/Lib/test/test_normalization.py b/Lib/test/test_normalization.py index 3040a086187..1c45ad5c314 100644 --- a/Lib/test/test_normalization.py +++ b/Lib/test/test_normalization.py @@ -57,7 +57,7 @@ def test_main(self): c1,c2,c3,c4,c5 = [unistr(x) for x in line.split(';')[:-1]] except RangeError: # Skip unsupported characters; - # try atleast adding c1 if we are in part1 + # try at least adding c1 if we are in part1 if part == "@Part1": try: c1 = unistr(line.split(';')[0]) diff --git a/Lib/test/test_openpty.py b/Lib/test/test_openpty.py index 20c4fe239c2..4b34b3a3c77 100644 --- a/Lib/test/test_openpty.py +++ b/Lib/test/test_openpty.py @@ -10,6 +10,8 @@ class OpenptyTest(unittest.TestCase): def test(self): master, slave = os.openpty() + self.addCleanup(os.close, master) + self.addCleanup(os.close, slave) if not os.isatty(slave): self.fail("Slave-end of pty is not a terminal.") diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index bdbeb0107bb..1d7e836f5c9 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -10,8 +10,13 @@ import signal import subprocess import time +try: + import resource +except ImportError: + resource = None from test import test_support +from test.script_helper import assert_python_ok import mmap import uuid @@ -563,9 +568,36 @@ def test_urandom_subprocess(self): data2 = self.get_urandom_subprocess(16) self.assertNotEqual(data1, data2) + @unittest.skipUnless(resource, "test requires the resource module") + def test_urandom_failure(self): + # Check urandom() failing when it is not able to open /dev/random. + # We spawn a new process to make the test more robust (if getrlimit() + # failed to restore the file descriptor limit after this, the whole + # test suite would crash; this actually happened on the OS X Tiger + # buildbot). + code = """if 1: + import errno + import os + import resource + + soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) + resource.setrlimit(resource.RLIMIT_NOFILE, (1, hard_limit)) + try: + os.urandom(16) + except OSError as e: + assert e.errno == errno.EMFILE, e.errno + else: + raise AssertionError("OSError not raised") + """ + assert_python_ok('-c', code) + + +class ExecvpeTests(unittest.TestCase): + def test_execvpe_with_bad_arglist(self): self.assertRaises(ValueError, os.execvpe, 'notepad', [], None) + class Win32ErrorTests(unittest.TestCase): def test_rename(self): self.assertRaises(WindowsError, os.rename, test_support.TESTFN, test_support.TESTFN+".bak") @@ -852,6 +884,7 @@ def test_main(): MakedirTests, DevNullTests, URandomTests, + ExecvpeTests, Win32ErrorTests, TestInvalidFD, PosixUidGidTests, diff --git a/Lib/test/test_pep263.py b/Lib/test/test_pep263.py index 9286467adf3..4b60624be21 100644 --- a/Lib/test/test_pep263.py +++ b/Lib/test/test_pep263.py @@ -41,6 +41,24 @@ def test_issue7820(self): # two bytes in common with the UTF-8 BOM self.assertRaises(SyntaxError, eval, '\xef\xbb\x20') + def test_error_message(self): + compile('# -*- coding: iso-8859-15 -*-\n', 'dummy', 'exec') + compile('\xef\xbb\xbf\n', 'dummy', 'exec') + compile('\xef\xbb\xbf# -*- coding: utf-8 -*-\n', 'dummy', 'exec') + with self.assertRaisesRegexp(SyntaxError, 'fake'): + compile('# -*- coding: fake -*-\n', 'dummy', 'exec') + with self.assertRaisesRegexp(SyntaxError, 'iso-8859-15'): + compile('\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n', + 'dummy', 'exec') + with self.assertRaisesRegexp(SyntaxError, 'BOM'): + compile('\xef\xbb\xbf# -*- coding: iso-8859-15 -*-\n', + 'dummy', 'exec') + with self.assertRaisesRegexp(SyntaxError, 'fake'): + compile('\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec') + with self.assertRaisesRegexp(SyntaxError, 'BOM'): + compile('\xef\xbb\xbf# -*- coding: fake -*-\n', 'dummy', 'exec') + + def test_main(): test_support.run_unittest(PEP263Test) diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index e6e0f868d3b..7859ad05723 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -135,6 +135,18 @@ def test_string(self): data2 = plistlib.writePlistToString(pl2) self.assertEqual(data, data2) + def test_indentation_array(self): + data = [[[[[[[[{'test': plistlib.Data(b'aaaaaa')}]]]]]]]] + self.assertEqual(plistlib.readPlistFromString(plistlib.writePlistToString(data)), data) + + def test_indentation_dict(self): + data = {'1': {'2': {'3': {'4': {'5': {'6': {'7': {'8': {'9': plistlib.Data(b'aaaaaa')}}}}}}}}} + self.assertEqual(plistlib.readPlistFromString(plistlib.writePlistToString(data)), data) + + def test_indentation_dict_mix(self): + data = {'1': {'2': [{'3': [[[[[{'test': plistlib.Data(b'aaaaaa')}]]]]]}]}} + self.assertEqual(plistlib.readPlistFromString(plistlib.writePlistToString(data)), data) + def test_appleformatting(self): pl = plistlib.readPlistFromString(TESTDATA) data = plistlib.writePlistToString(pl) diff --git a/Lib/test/test_poll.py b/Lib/test/test_poll.py index 55294f8c229..219fa0e4647 100644 --- a/Lib/test/test_poll.py +++ b/Lib/test/test_poll.py @@ -1,8 +1,16 @@ # Test case for the os.poll() function -import os, select, random, unittest +import os +import random +import select import _testcapi -from test.test_support import TESTFN, run_unittest +try: + import threading +except ImportError: + threading = None +import time +import unittest +from test.test_support import TESTFN, run_unittest, reap_threads try: select.poll @@ -160,6 +168,36 @@ def test_poll3(self): self.assertRaises(OverflowError, pollster.poll, _testcapi.INT_MAX + 1) self.assertRaises(OverflowError, pollster.poll, _testcapi.UINT_MAX + 1) + @unittest.skipUnless(threading, 'Threading required for this test.') + @reap_threads + def test_threaded_poll(self): + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + rfds = [] + for i in range(10): + fd = os.dup(r) + self.addCleanup(os.close, fd) + rfds.append(fd) + pollster = select.poll() + for fd in rfds: + pollster.register(fd, select.POLLIN) + + t = threading.Thread(target=pollster.poll) + t.start() + try: + time.sleep(0.5) + # trigger ufds array reallocation + for fd in rfds: + pollster.unregister(fd) + pollster.register(w, select.POLLOUT) + self.assertRaises(RuntimeError, pollster.poll) + finally: + # and make the call to poll() from the thread return + os.write(w, b'spam') + t.join() + + def test_main(): run_unittest(PollTests) diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index e0137dfc59f..50493f6d1a8 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -24,6 +24,20 @@ class tuple3(tuple): def __repr__(self): return tuple.__repr__(self) +class set2(set): + pass + +class set3(set): + def __repr__(self): + return set.__repr__(self) + +class frozenset2(frozenset): + pass + +class frozenset3(frozenset): + def __repr__(self): + return frozenset.__repr__(self) + class dict2(dict): pass @@ -114,22 +128,24 @@ def test_same_as_repr(self): for simple in (0, 0L, 0+0j, 0.0, "", uni(""), (), tuple2(), tuple3(), [], list2(), list3(), + set(), set2(), set3(), + frozenset(), frozenset2(), frozenset3(), {}, dict2(), dict3(), self.assertTrue, pprint, -6, -6L, -6-6j, -1.5, "x", uni("x"), (3,), [3], {3: 6}, (1,2), [3,4], {5: 6}, tuple2((1,2)), tuple3((1,2)), tuple3(range(100)), [3,4], list2([3,4]), list3([3,4]), list3(range(100)), + set({7}), set2({7}), set3({7}), + frozenset({8}), frozenset2({8}), frozenset3({8}), dict2({5: 6}), dict3({5: 6}), range(10, -11, -1) ): native = repr(simple) - for function in "pformat", "saferepr": - f = getattr(pprint, function) - got = f(simple) - self.assertEqual(native, got, - "expected %s got %s from pprint.%s" % - (native, got, function)) + self.assertEqual(pprint.pformat(simple), native) + self.assertEqual(pprint.pformat(simple, width=1, indent=0) + .replace('\n', ' '), native) + self.assertEqual(pprint.saferepr(simple), native) def test_basic_line_wrap(self): # verify basic line-wrapping operation @@ -205,19 +221,59 @@ def test_subclassing(self): self.assertEqual(DottedPrettyPrinter().pformat(o), exp) def test_set_reprs(self): - self.assertEqual(pprint.pformat(set()), 'set()') + self.assertEqual(pprint.pformat(set()), 'set([])') self.assertEqual(pprint.pformat(set(range(3))), 'set([0, 1, 2])') - self.assertEqual(pprint.pformat(frozenset()), 'frozenset()') - self.assertEqual(pprint.pformat(frozenset(range(3))), 'frozenset([0, 1, 2])') + self.assertEqual(pprint.pformat(set(range(7)), width=20), '''\ +set([0, + 1, + 2, + 3, + 4, + 5, + 6])''') + self.assertEqual(pprint.pformat(set2(range(7)), width=20), '''\ +set2([0, + 1, + 2, + 3, + 4, + 5, + 6])''') + self.assertEqual(pprint.pformat(set3(range(7)), width=20), + 'set3([0, 1, 2, 3, 4, 5, 6])') + + self.assertEqual(pprint.pformat(frozenset()), 'frozenset([])') + self.assertEqual(pprint.pformat(frozenset(range(3))), + 'frozenset([0, 1, 2])') + self.assertEqual(pprint.pformat(frozenset(range(7)), width=20), '''\ +frozenset([0, + 1, + 2, + 3, + 4, + 5, + 6])''') + self.assertEqual(pprint.pformat(frozenset2(range(7)), width=20), '''\ +frozenset2([0, + 1, + 2, + 3, + 4, + 5, + 6])''') + self.assertEqual(pprint.pformat(frozenset3(range(7)), width=20), + 'frozenset3([0, 1, 2, 3, 4, 5, 6])') + + def test_set_of_sets_reprs(self): cube_repr_tgt = """\ {frozenset([]): frozenset([frozenset([2]), frozenset([0]), frozenset([1])]), - frozenset([0]): frozenset([frozenset(), + frozenset([0]): frozenset([frozenset([]), frozenset([0, 2]), frozenset([0, 1])]), - frozenset([1]): frozenset([frozenset(), + frozenset([1]): frozenset([frozenset([]), frozenset([1, 2]), frozenset([0, 1])]), - frozenset([2]): frozenset([frozenset(), + frozenset([2]): frozenset([frozenset([]), frozenset([1, 2]), frozenset([0, 2])]), frozenset([1, 2]): frozenset([frozenset([2]), @@ -243,7 +299,7 @@ def test_set_reprs(self): frozenset([frozenset([0]), frozenset([0, 1])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([0])]), frozenset([frozenset([2]), frozenset([0, @@ -259,7 +315,7 @@ def test_set_reprs(self): frozenset([frozenset([1]), frozenset([1, 2])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([1])])]), frozenset([frozenset([1, 2]), frozenset([1])]): frozenset([frozenset([frozenset([1, 2]), @@ -269,7 +325,7 @@ def test_set_reprs(self): frozenset([frozenset([2]), frozenset([1, 2])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([1])]), frozenset([frozenset([1]), frozenset([0, @@ -285,7 +341,7 @@ def test_set_reprs(self): frozenset([frozenset([2]), frozenset([0, 2])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([2])])]), frozenset([frozenset([]), frozenset([0])]): frozenset([frozenset([frozenset([0]), frozenset([0, @@ -293,16 +349,16 @@ def test_set_reprs(self): frozenset([frozenset([0]), frozenset([0, 2])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([1])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([2])])]), - frozenset([frozenset([]), frozenset([1])]): frozenset([frozenset([frozenset(), + frozenset([frozenset([]), frozenset([1])]): frozenset([frozenset([frozenset([]), frozenset([0])]), frozenset([frozenset([1]), frozenset([1, 2])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([2])]), frozenset([frozenset([1]), frozenset([0, @@ -310,9 +366,9 @@ def test_set_reprs(self): frozenset([frozenset([2]), frozenset([])]): frozenset([frozenset([frozenset([2]), frozenset([1, 2])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([0])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([1])]), frozenset([frozenset([2]), frozenset([0, @@ -333,7 +389,7 @@ def test_set_reprs(self): frozenset([frozenset([1]), frozenset([0, 1])])]), - frozenset([frozenset([0]), frozenset([0, 1])]): frozenset([frozenset([frozenset(), + frozenset([frozenset([0]), frozenset([0, 1])]): frozenset([frozenset([frozenset([]), frozenset([0])]), frozenset([frozenset([0, 1]), @@ -357,7 +413,7 @@ def test_set_reprs(self): frozenset([frozenset([0]), frozenset([0, 2])]), - frozenset([frozenset(), + frozenset([frozenset([]), frozenset([2])])]), frozenset([frozenset([0, 1, 2]), frozenset([0, 2])]): frozenset([frozenset([frozenset([1, 2]), diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index 382d6dc878f..9e62f2859fe 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -4,15 +4,17 @@ import __builtin__ import re import pydoc +import contextlib import inspect import keyword +import pkgutil import unittest import xml.etree import test.test_support from collections import namedtuple from test.script_helper import assert_python_ok from test.test_support import ( - TESTFN, rmtree, reap_children, captured_stdout) + TESTFN, rmtree, reap_children, captured_stdout, captured_stderr) from test import pydoc_mod @@ -228,7 +230,30 @@ def print_diffs(text1, text2): print '\n' + ''.join(diffs) -class PyDocDocTest(unittest.TestCase): +class PydocBaseTest(unittest.TestCase): + + def _restricted_walk_packages(self, walk_packages, path=None): + """ + A version of pkgutil.walk_packages() that will restrict itself to + a given path. + """ + default_path = path or [os.path.dirname(__file__)] + def wrapper(path=None, prefix='', onerror=None): + return walk_packages(path or default_path, prefix, onerror) + return wrapper + + @contextlib.contextmanager + def restrict_walk_packages(self, path=None): + walk_packages = pkgutil.walk_packages + pkgutil.walk_packages = self._restricted_walk_packages(walk_packages, + path) + try: + yield + finally: + pkgutil.walk_packages = walk_packages + + +class PydocDocTest(unittest.TestCase): @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") @@ -303,7 +328,7 @@ def test_stripid(self): "") -class PydocImportTest(unittest.TestCase): +class PydocImportTest(PydocBaseTest): def setUp(self): self.test_dir = os.mkdir(TESTFN) @@ -338,8 +363,19 @@ def test_apropos_with_bad_package(self): badsyntax = os.path.join(pkgdir, "__init__") + os.extsep + "py" with open(badsyntax, 'w') as f: f.write("invalid python syntax = $1\n") - result = run_pydoc('zqwykjv', '-k', PYTHONPATH=TESTFN) - self.assertEqual('', result) + with self.restrict_walk_packages(path=[TESTFN]): + with captured_stdout() as out: + with captured_stderr() as err: + pydoc.apropos('xyzzy') + # No result, no error + self.assertEqual(out.getvalue(), '') + self.assertEqual(err.getvalue(), '') + # The package name is still matched + with captured_stdout() as out: + with captured_stderr() as err: + pydoc.apropos('syntaxerr') + self.assertEqual(out.getvalue().strip(), 'syntaxerr') + self.assertEqual(err.getvalue(), '') def test_apropos_with_unreadable_dir(self): # Issue 7367 - pydoc -k failed when unreadable dir on path @@ -348,8 +384,13 @@ def test_apropos_with_unreadable_dir(self): self.addCleanup(os.rmdir, self.unreadable_dir) # Note, on Windows the directory appears to be still # readable so this is not really testing the issue there - result = run_pydoc('zqwykjv', '-k', PYTHONPATH=TESTFN) - self.assertEqual('', result) + with self.restrict_walk_packages(path=[TESTFN]): + with captured_stdout() as out: + with captured_stderr() as err: + pydoc.apropos('SOMEKEY') + # No result, no error + self.assertEqual(out.getvalue(), '') + self.assertEqual(err.getvalue(), '') class TestDescriptions(unittest.TestCase): @@ -412,7 +453,7 @@ def test_builtin(self): def test_main(): try: - test.test_support.run_unittest(PyDocDocTest, + test.test_support.run_unittest(PydocDocTest, PydocImportTest, TestDescriptions, TestHelper) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index 6ddddda9e1b..8b277cfc02e 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2,6 +2,7 @@ from test.test_support import precisionbigmemtest, _2G, cpython_only import re from re import Scanner +import sre_constants import sys import string import traceback @@ -886,6 +887,36 @@ def test_repeat_minmax_overflow_maxrepeat(self): self.assertRaises(OverflowError, re.compile, r".{,%d}" % MAXREPEAT) self.assertRaises(OverflowError, re.compile, r".{%d,}?" % MAXREPEAT) + def test_backref_group_name_in_exception(self): + # Issue 17341: Poor error message when compiling invalid regex + with self.assertRaisesRegexp(sre_constants.error, ''): + re.compile('(?P=)') + + def test_group_name_in_exception(self): + # Issue 17341: Poor error message when compiling invalid regex + with self.assertRaisesRegexp(sre_constants.error, '\?foo'): + re.compile('(?P)') + + def test_issue17998(self): + for reps in '*', '+', '?', '{1}': + for mod in '', '?': + pattern = '.' + reps + mod + 'yz' + self.assertEqual(re.compile(pattern, re.S).findall('xyz'), + ['xyz'], msg=pattern) + pattern = pattern.encode() + self.assertEqual(re.compile(pattern, re.S).findall(b'xyz'), + [b'xyz'], msg=pattern) + + + def test_bug_2537(self): + # issue 2537: empty submatches + for outer_op in ('{0,}', '*', '+', '{1,187}'): + for inner_op in ('{0,}', '*', '?'): + r = re.compile("^((x|y)%s)%s" % (inner_op, outer_op)) + m = r.match("xyyzy") + self.assertEqual(m.group(0), "xyy") + self.assertEqual(m.group(1), "") + self.assertEqual(m.group(2), "y") def run_re_tests(): from test.re_tests import tests, SUCCEED, FAIL, SYNTAX_ERROR diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index b3d4a46056b..651301bde2a 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -228,6 +228,18 @@ def RobotTest(index, robots_txt, good_urls, bad_urls, RobotTest(15, doc, good, bad) +# 16. Empty query (issue #17403). Normalizing the url first. +doc = """ +User-agent: * +Allow: /some/path? +Disallow: /another/path? +""" + +good = ['/some/path?'] +bad = ['/another/path?'] + +RobotTest(16, doc, good, bad) + class NetworkTestCase(unittest.TestCase): diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index 3a20eac6b12..86638a2665d 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -284,6 +284,26 @@ def test_xmlgen_ignorable(self): self.assertEqual(result.getvalue(), start + " ") + def test_xmlgen_encoding_bytes(self): + encodings = ('iso-8859-15', 'utf-8', + 'utf-16be', 'utf-16le', + 'utf-32be', 'utf-32le') + for encoding in encodings: + result = self.ioclass() + gen = XMLGenerator(result, encoding=encoding) + + gen.startDocument() + gen.startElement("doc", {"a": u'\u20ac'}) + gen.characters(u"\u20ac".encode(encoding)) + gen.ignorableWhitespace(" ".encode(encoding)) + gen.endElement("doc") + gen.endDocument() + + self.assertEqual(result.getvalue(), ( + u'\n' + u'\u20ac ' % encoding + ).encode(encoding, 'xmlcharrefreplace')) + def test_xmlgen_ns(self): result = self.ioclass() gen = XMLGenerator(result) diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 111e55394a6..aef9b1a95d2 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -665,6 +665,9 @@ def testGetaddrinfo(self): socket.getaddrinfo(None, 0, socket.AF_UNSPEC, socket.SOCK_STREAM, 0, socket.AI_PASSIVE) + # Issue 17269 + if hasattr(socket, 'AI_NUMERICSERV'): + socket.getaddrinfo("localhost", None, 0, 0, 0, socket.AI_NUMERICSERV) def check_sendall_interrupted(self, with_timeout): # socketpair() is not stricly required, but it makes things easier. @@ -685,11 +688,12 @@ def raising_handler(*args): c.settimeout(1.5) with self.assertRaises(ZeroDivisionError): signal.alarm(1) - c.sendall(b"x" * (1024**2)) + c.sendall(b"x" * test_support.SOCK_MAX_SIZE) if with_timeout: signal.signal(signal.SIGALRM, ok_handler) signal.alarm(1) - self.assertRaises(socket.timeout, c.sendall, b"x" * (1024**2)) + self.assertRaises(socket.timeout, c.sendall, + b"x" * test_support.SOCK_MAX_SIZE) finally: signal.signal(signal.SIGALRM, old_alarm) c.close() diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 75521990228..9c33c8f421a 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -25,6 +25,7 @@ HOST = test_support.HOST CERTFILE = None SVN_PYTHON_ORG_ROOT_CERT = None +NULLBYTECERT = None def handle_error(prefix): exc_format = ' '.join(traceback.format_exception(*sys.exc_info())) @@ -123,6 +124,35 @@ def test_parse_cert(self): ('DNS', 'projects.forum.nokia.com')) ) + def test_parse_cert_CVE_2013_4238(self): + p = ssl._ssl._test_decode_cert(NULLBYTECERT) + if test_support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + subject = ((('countryName', 'US'),), + (('stateOrProvinceName', 'Oregon'),), + (('localityName', 'Beaverton'),), + (('organizationName', 'Python Software Foundation'),), + (('organizationalUnitName', 'Python Core Development'),), + (('commonName', 'null.python.org\x00example.org'),), + (('emailAddress', 'python-dev@python.org'),)) + self.assertEqual(p['subject'], subject) + self.assertEqual(p['issuer'], subject) + if ssl.OPENSSL_VERSION_INFO >= (0, 9, 8): + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '2001:DB8:0:0:0:0:0:1\n')) + else: + # OpenSSL 0.9.7 doesn't support IPv6 addresses in subjectAltName + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '')) + + self.assertEqual(p['subjectAltName'], san) + def test_DER_to_PEM(self): with open(SVN_PYTHON_ORG_ROOT_CERT, 'r') as f: pem = f.read() @@ -1360,7 +1390,7 @@ def test_default_ciphers(self): def test_main(verbose=False): - global CERTFILE, SVN_PYTHON_ORG_ROOT_CERT, NOKIACERT + global CERTFILE, SVN_PYTHON_ORG_ROOT_CERT, NOKIACERT, NULLBYTECERT CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "keycert.pem") SVN_PYTHON_ORG_ROOT_CERT = os.path.join( @@ -1368,10 +1398,13 @@ def test_main(verbose=False): "https_svn_python_org_root.pem") NOKIACERT = os.path.join(os.path.dirname(__file__) or os.curdir, "nokia.pem") + NULLBYTECERT = os.path.join(os.path.dirname(__file__) or os.curdir, + "nullbytecert.pem") if (not os.path.exists(CERTFILE) or not os.path.exists(SVN_PYTHON_ORG_ROOT_CERT) or - not os.path.exists(NOKIACERT)): + not os.path.exists(NOKIACERT) or + not os.path.exists(NULLBYTECERT)): raise test_support.TestFailed("Can't read certificate files!") tests = [BasicTests, BasicSocketTests] diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py new file mode 100644 index 00000000000..a71f599b8d6 --- /dev/null +++ b/Lib/test/test_stat.py @@ -0,0 +1,175 @@ +import unittest +import os +from test.test_support import TESTFN, run_unittest +import stat + +class TestFilemode(unittest.TestCase): + file_flags = {'SF_APPEND', 'SF_ARCHIVED', 'SF_IMMUTABLE', 'SF_NOUNLINK', + 'SF_SNAPSHOT', 'UF_APPEND', 'UF_COMPRESSED', 'UF_HIDDEN', + 'UF_IMMUTABLE', 'UF_NODUMP', 'UF_NOUNLINK', 'UF_OPAQUE'} + + formats = {'S_IFBLK', 'S_IFCHR', 'S_IFDIR', 'S_IFIFO', 'S_IFLNK', + 'S_IFREG', 'S_IFSOCK'} + + format_funcs = {'S_ISBLK', 'S_ISCHR', 'S_ISDIR', 'S_ISFIFO', 'S_ISLNK', + 'S_ISREG', 'S_ISSOCK'} + + stat_struct = { + 'ST_MODE': 0, + 'ST_INO': 1, + 'ST_DEV': 2, + 'ST_NLINK': 3, + 'ST_UID': 4, + 'ST_GID': 5, + 'ST_SIZE': 6, + 'ST_ATIME': 7, + 'ST_MTIME': 8, + 'ST_CTIME': 9} + + # permission bit value are defined by POSIX + permission_bits = { + 'S_ISUID': 0o4000, + 'S_ISGID': 0o2000, + 'S_ENFMT': 0o2000, + 'S_ISVTX': 0o1000, + 'S_IRWXU': 0o700, + 'S_IRUSR': 0o400, + 'S_IREAD': 0o400, + 'S_IWUSR': 0o200, + 'S_IWRITE': 0o200, + 'S_IXUSR': 0o100, + 'S_IEXEC': 0o100, + 'S_IRWXG': 0o070, + 'S_IRGRP': 0o040, + 'S_IWGRP': 0o020, + 'S_IXGRP': 0o010, + 'S_IRWXO': 0o007, + 'S_IROTH': 0o004, + 'S_IWOTH': 0o002, + 'S_IXOTH': 0o001} + + def setUp(self): + try: + os.remove(TESTFN) + except OSError: + try: + os.rmdir(TESTFN) + except OSError: + pass + tearDown = setUp + + def get_mode(self, fname=TESTFN, lstat=True): + if lstat: + st_mode = os.lstat(fname).st_mode + else: + st_mode = os.stat(fname).st_mode + return st_mode + + def assertS_IS(self, name, mode): + # test format, lstrip is for S_IFIFO + fmt = getattr(stat, "S_IF" + name.lstrip("F")) + self.assertEqual(stat.S_IFMT(mode), fmt) + # test that just one function returns true + testname = "S_IS" + name + for funcname in self.format_funcs: + func = getattr(stat, funcname, None) + if func is None: + if funcname == testname: + raise ValueError(funcname) + continue + if funcname == testname: + self.assertTrue(func(mode)) + else: + self.assertFalse(func(mode)) + + def test_mode(self): + with open(TESTFN, 'w'): + pass + if os.name == 'posix': + os.chmod(TESTFN, 0o700) + st_mode = self.get_mode() + self.assertS_IS("REG", st_mode) + self.assertEqual(stat.S_IMODE(st_mode), + stat.S_IRWXU) + + os.chmod(TESTFN, 0o070) + st_mode = self.get_mode() + self.assertS_IS("REG", st_mode) + self.assertEqual(stat.S_IMODE(st_mode), + stat.S_IRWXG) + + os.chmod(TESTFN, 0o007) + st_mode = self.get_mode() + self.assertS_IS("REG", st_mode) + self.assertEqual(stat.S_IMODE(st_mode), + stat.S_IRWXO) + + os.chmod(TESTFN, 0o444) + st_mode = self.get_mode() + self.assertS_IS("REG", st_mode) + self.assertEqual(stat.S_IMODE(st_mode), 0o444) + else: + os.chmod(TESTFN, 0o700) + st_mode = self.get_mode() + self.assertS_IS("REG", st_mode) + self.assertEqual(stat.S_IFMT(st_mode), + stat.S_IFREG) + + def test_directory(self): + os.mkdir(TESTFN) + os.chmod(TESTFN, 0o700) + st_mode = self.get_mode() + self.assertS_IS("DIR", st_mode) + + @unittest.skipUnless(hasattr(os, 'symlink'), 'os.symlink not available') + def test_link(self): + try: + os.symlink(os.getcwd(), TESTFN) + except (OSError, NotImplementedError) as err: + raise unittest.SkipTest(str(err)) + else: + st_mode = self.get_mode() + self.assertS_IS("LNK", st_mode) + + @unittest.skipUnless(hasattr(os, 'mkfifo'), 'os.mkfifo not available') + def test_fifo(self): + os.mkfifo(TESTFN, 0o700) + st_mode = self.get_mode() + self.assertS_IS("FIFO", st_mode) + + @unittest.skipUnless(os.name == 'posix', 'requires Posix') + def test_devices(self): + if os.path.exists(os.devnull): + st_mode = self.get_mode(os.devnull, lstat=False) + self.assertS_IS("CHR", st_mode) + # Linux block devices, BSD has no block devices anymore + for blockdev in ("/dev/sda", "/dev/hda"): + if os.path.exists(blockdev): + st_mode = self.get_mode(blockdev, lstat=False) + self.assertS_IS("BLK", st_mode) + break + + def test_module_attributes(self): + for key, value in self.stat_struct.items(): + modvalue = getattr(stat, key) + self.assertEqual(value, modvalue, key) + for key, value in self.permission_bits.items(): + modvalue = getattr(stat, key) + self.assertEqual(value, modvalue, key) + for key in self.file_flags: + modvalue = getattr(stat, key) + self.assertIsInstance(modvalue, int) + for key in self.formats: + modvalue = getattr(stat, key) + self.assertIsInstance(modvalue, int) + for key in self.format_funcs: + func = getattr(stat, key) + self.assertTrue(callable(func)) + self.assertEqual(func(0), 0) + + +def test_main(): + run_unittest(TestFilemode) + +if __name__ == '__main__': + test_main() diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index e89d84f4903..0efcdbf2534 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -14,6 +14,10 @@ import resource except ImportError: resource = None +try: + import threading +except ImportError: + threading = None mswindows = (sys.platform == "win32") @@ -629,6 +633,36 @@ def test_leaking_fds_on_error(self): if c.exception.errno not in (errno.ENOENT, errno.EACCES): raise c.exception + @unittest.skipIf(threading is None, "threading required") + def test_double_close_on_error(self): + # Issue #18851 + fds = [] + def open_fds(): + for i in range(20): + fds.extend(os.pipe()) + time.sleep(0.001) + t = threading.Thread(target=open_fds) + t.start() + try: + with self.assertRaises(EnvironmentError): + subprocess.Popen(['nonexisting_i_hope'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + finally: + t.join() + exc = None + for fd in fds: + # If a double close occurred, some of those fds will + # already have been closed by mistake, and os.close() + # here will raise. + try: + os.close(fd) + except OSError as e: + exc = e + if exc is not None: + raise exc + def test_handles_closed_on_exception(self): # If CreateProcess exits with an error, ensure the # duplicate output handles are released @@ -783,7 +817,7 @@ def __init__(self, testcase, *args, **kwargs): def _execute_child( self, args, executable, preexec_fn, close_fds, cwd, env, - universal_newlines, startupinfo, creationflags, shell, + universal_newlines, startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite): @@ -791,7 +825,7 @@ def _execute_child( subprocess.Popen._execute_child( self, args, executable, preexec_fn, close_fds, cwd, env, universal_newlines, - startupinfo, creationflags, shell, + startupinfo, creationflags, shell, to_close, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite) @@ -806,7 +840,8 @@ def _execute_child( self._testcase.assertNotIn( fd, (p2cwrite, c2pread, errread)) finally: - map(os.close, devzero_fds) + for fd in devzero_fds: + os.close(fd) @unittest.skipIf(not os.path.exists("/dev/zero"), "/dev/zero required.") def test_preexec_errpipe_does_not_double_close_pipes(self): diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 33de7881803..1f3c039b2f5 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -184,7 +184,7 @@ def unload(name): if sys.platform.startswith("win"): def _waitfor(func, pathname, waitall=False): - # Peform the operation + # Perform the operation func(pathname) # Now setup the wait loop if waitall: @@ -200,7 +200,7 @@ def _waitfor(func, pathname, waitall=False): # required when contention occurs. timeout = 0.001 while timeout < 1.0: - # Note we are only testing for the existance of the file(s) in + # Note we are only testing for the existence of the file(s) in # the contents of the directory regardless of any security or # access rights. If we have made it this far, we have sufficient # permissions to do that much using Python's equivalent of the @@ -290,7 +290,12 @@ def requires(resource, msg=None): msg = "Use of the `%s' resource not enabled" % resource raise ResourceDenied(msg) -HOST = 'localhost' + +# Don't use "localhost", since resolving it uses the DNS under recent +# Windows versions (see issue #18792). +HOST = "127.0.0.1" +HOSTv6 = "::1" + def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): """Returns an unused port that should be suitable for binding. This is @@ -400,6 +405,21 @@ def fcmp(x, y): # fuzzy comparison function return (len(x) > len(y)) - (len(x) < len(y)) return (x > y) - (x < y) + +# A constant likely larger than the underlying OS pipe buffer size, to +# make writes blocking. +# Windows limit seems to be around 512 B, and many Unix kernels have a +# 64 KiB pipe buffer size or 16 * PAGE_SIZE: take a few megs to be sure. +# (see issue #17835 for a discussion of this number). +PIPE_MAX_SIZE = 4 * 1024 * 1024 + 1 + +# A constant likely larger than the underlying OS socket buffer size, to make +# writes blocking. +# The socket buffer sizes can usually be tuned system-wide (e.g. through sysctl +# on Linux), or on a per-socket basis (SO_SNDBUF/SO_RCVBUF). See issue #18643 +# for a discussion of this number). +SOCK_MAX_SIZE = 16 * 1024 * 1024 + 1 + try: unicode have_unicode = True diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index af59e277f3e..69d342a5031 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -345,6 +345,14 @@ def test_init_close_fobj(self): finally: os.remove(empty) + def test_parallel_iteration(self): + # Issue #16601: Restarting iteration over tarfile continued + # from where it left off. + with tarfile.open(self.tarname) as tar: + for m1, m2 in zip(tar, tar): + self.assertEqual(m1.offset, m2.offset) + self.assertEqual(m1.name, m2.name) + class StreamReadTest(CommonReadTest): diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index 8db11dc77d0..50e1ac40acf 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -3,7 +3,9 @@ import unittest import sys import os +import _testcapi from test import test_support +from subprocess import Popen, PIPE # Skip this test if the _tkinter module wasn't built. _tkinter = test_support.import_module('_tkinter') @@ -11,6 +13,14 @@ from Tkinter import Tcl from _tkinter import TclError +tcl_version = _tkinter.TCL_VERSION.split('.') +try: + for i in range(len(tcl_version)): + tcl_version[i] = int(tcl_version[i]) +except ValueError: + pass +tcl_version = tuple(tcl_version) + class TkinterTest(unittest.TestCase): @@ -146,11 +156,20 @@ def testLoadWithUNC(self): with test_support.EnvironmentVarGuard() as env: env.unset("TCL_LIBRARY") - f = os.popen('%s -c "import Tkinter; print Tkinter"' % (unc_name,)) + cmd = '%s -c "import Tkinter; print Tkinter"' % (unc_name,) + + p = Popen(cmd, stdout=PIPE, stderr=PIPE) + out_data, err_data = p.communicate() + + msg = '\n\n'.join(['"Tkinter.py" not in output', + 'Command:', cmd, + 'stdout:', out_data, + 'stderr:', err_data]) + + self.assertIn('Tkinter.py', out_data, msg) + + self.assertEqual(p.wait(), 0, 'Non-zero exit code') - self.assertTrue('Tkinter.py' in f.read()) - # exit code must be zero - self.assertEqual(f.close(), None) def test_passing_values(self): def passValue(value): @@ -174,9 +193,93 @@ def passValue(value): self.assertEqual(passValue(f), f) self.assertEqual(passValue((1, '2', (3.4,))), (1, '2', (3.4,))) + def test_splitlist(self): + splitlist = self.interp.tk.splitlist + call = self.interp.tk.call + self.assertRaises(TypeError, splitlist) + self.assertRaises(TypeError, splitlist, 'a', 'b') + self.assertRaises(TypeError, splitlist, 2) + testcases = [ + ('2', ('2',)), + ('', ()), + ('{}', ('',)), + ('""', ('',)), + ('a\n b\t\r c\n ', ('a', 'b', 'c')), + (u'a\n b\t\r c\n ', ('a', 'b', 'c')), + ('a \xe2\x82\xac', ('a', '\xe2\x82\xac')), + (u'a \u20ac', ('a', '\xe2\x82\xac')), + ('a {b c}', ('a', 'b c')), + (r'a b\ c', ('a', 'b c')), + (('a', 'b c'), ('a', 'b c')), + ('a 2', ('a', '2')), + (('a', 2), ('a', 2)), + ('a 3.4', ('a', '3.4')), + (('a', 3.4), ('a', 3.4)), + ((), ()), + (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), + ] + if tcl_version >= (8, 5): + testcases += [ + (call('dict', 'create', 1, u'\u20ac', '\xe2\x82\xac', (3.4,)), + (1, u'\u20ac', u'\u20ac', (3.4,))), + ] + for arg, res in testcases: + self.assertEqual(splitlist(arg), res) + self.assertRaises(TclError, splitlist, '{') + + def test_split(self): + split = self.interp.tk.split + call = self.interp.tk.call + self.assertRaises(TypeError, split) + self.assertRaises(TypeError, split, 'a', 'b') + self.assertRaises(TypeError, split, 2) + testcases = [ + ('2', '2'), + ('', ''), + ('{}', ''), + ('""', ''), + ('{', '{'), + ('a\n b\t\r c\n ', ('a', 'b', 'c')), + (u'a\n b\t\r c\n ', ('a', 'b', 'c')), + ('a \xe2\x82\xac', ('a', '\xe2\x82\xac')), + (u'a \u20ac', ('a', '\xe2\x82\xac')), + ('a {b c}', ('a', ('b', 'c'))), + (r'a b\ c', ('a', ('b', 'c'))), + (('a', 'b c'), ('a', ('b', 'c'))), + (('a', u'b c'), ('a', ('b', 'c'))), + ('a 2', ('a', '2')), + (('a', 2), ('a', 2)), + ('a 3.4', ('a', '3.4')), + (('a', 3.4), ('a', 3.4)), + (('a', (2, 3.4)), ('a', (2, 3.4))), + ((), ()), + (call('list', 1, '2', (3.4,)), (1, '2', (3.4,))), + ] + if tcl_version >= (8, 5): + testcases += [ + (call('dict', 'create', 12, u'\u20ac', '\xe2\x82\xac', (3.4,)), + (12, u'\u20ac', u'\u20ac', (3.4,))), + ] + for arg, res in testcases: + self.assertEqual(split(arg), res) + + +class BigmemTclTest(unittest.TestCase): + + def setUp(self): + self.interp = Tcl() + + @unittest.skipUnless(_testcapi.INT_MAX < _testcapi.PY_SSIZE_T_MAX, + "needs UINT_MAX < SIZE_MAX") + @test_support.precisionbigmemtest(size=_testcapi.INT_MAX + 1, memuse=5, + dry_run=False) + def test_huge_string(self, size): + value = ' ' * size + self.assertRaises(OverflowError, self.interp.call, 'set', '_', value) + def test_main(): - test_support.run_unittest(TclTest, TkinterTest) + test_support.run_unittest(TclTest, TkinterTest, BigmemTclTest) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_telnetlib.py b/Lib/test/test_telnetlib.py index 7fdb49e8e62..a5ed34bfa90 100644 --- a/Lib/test/test_telnetlib.py +++ b/Lib/test/test_telnetlib.py @@ -92,6 +92,14 @@ def testTimeoutOpen(self): self.assertEqual(telnet.sock.gettimeout(), 30) telnet.sock.close() + def testGetters(self): + # Test telnet getter methods + telnet = telnetlib.Telnet(HOST, self.port, timeout=30) + t_sock = telnet.sock + self.assertEqual(telnet.get_socket(), t_sock) + self.assertEqual(telnet.fileno(), t_sock.fileno()) + telnet.sock.close() + def _read_setUp(self): self.evt = threading.Event() self.dataq = Queue.Queue() diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index b4d23addf2b..011e7d0e1cc 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -8,6 +8,7 @@ import sys import re import warnings +import contextlib import unittest from test import test_support as support @@ -270,6 +271,22 @@ def test_same_thing(self): test_classes.append(test__get_candidate_names) +@contextlib.contextmanager +def _inside_empty_temp_dir(): + dir = tempfile.mkdtemp() + try: + with support.swap_attr(tempfile, 'tempdir', dir): + yield + finally: + support.rmtree(dir) + + +def _mock_candidate_names(*names): + return support.swap_attr(tempfile, + '_get_candidate_names', + lambda: iter(names)) + + class test__mkstemp_inner(TC): """Test the internal function _mkstemp_inner.""" @@ -386,6 +403,37 @@ def test_textmode(self): self.do_create(bin=0).write("blat\n") # XXX should test that the file really is a text file + def default_mkstemp_inner(self): + return tempfile._mkstemp_inner(tempfile.gettempdir(), + tempfile.template, + '', + tempfile._bin_openflags) + + def test_collision_with_existing_file(self): + # _mkstemp_inner tries another name when a file with + # the chosen name already exists + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + (fd1, name1) = self.default_mkstemp_inner() + os.close(fd1) + self.assertTrue(name1.endswith('aaa')) + + (fd2, name2) = self.default_mkstemp_inner() + os.close(fd2) + self.assertTrue(name2.endswith('bbb')) + + def test_collision_with_existing_directory(self): + # _mkstemp_inner tries another name when a directory with + # the chosen name already exists + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + dir = tempfile.mkdtemp() + self.assertTrue(dir.endswith('aaa')) + + (fd, name) = self.default_mkstemp_inner() + os.close(fd) + self.assertTrue(name.endswith('bbb')) + test_classes.append(test__mkstemp_inner) @@ -561,6 +609,27 @@ def test_mode(self): finally: os.rmdir(dir) + def test_collision_with_existing_file(self): + # mkdtemp tries another name when a file with + # the chosen name already exists + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + file = tempfile.NamedTemporaryFile(delete=False) + file.close() + self.assertTrue(file.name.endswith('aaa')) + dir = tempfile.mkdtemp() + self.assertTrue(dir.endswith('bbb')) + + def test_collision_with_existing_directory(self): + # mkdtemp tries another name when a directory with + # the chosen name already exists + with _inside_empty_temp_dir(), \ + _mock_candidate_names('aaa', 'aaa', 'bbb'): + dir1 = tempfile.mkdtemp() + self.assertTrue(dir1.endswith('aaa')) + dir2 = tempfile.mkdtemp() + self.assertTrue(dir2.endswith('bbb')) + test_classes.append(test_mkdtemp) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index ef04cd360a4..ac0b4bc701d 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -443,6 +443,46 @@ def background_thread(evt): self.assertEqual(out, '') self.assertEqual(err, '') + @unittest.skipUnless(hasattr(os, 'fork'), "needs os.fork()") + def test_is_alive_after_fork(self): + # Try hard to trigger #18418: is_alive() could sometimes be True on + # threads that vanished after a fork. + old_interval = sys.getcheckinterval() + + # Make the bug more likely to manifest. + sys.setcheckinterval(10) + + try: + for i in range(20): + t = threading.Thread(target=lambda: None) + t.start() + pid = os.fork() + if pid == 0: + os._exit(1 if t.is_alive() else 0) + else: + t.join() + pid, status = os.waitpid(pid, 0) + self.assertEqual(0, status) + finally: + sys.setcheckinterval(old_interval) + + def test_BoundedSemaphore_limit(self): + # BoundedSemaphore should raise ValueError if released too often. + for limit in range(1, 10): + bs = threading.BoundedSemaphore(limit) + threads = [threading.Thread(target=bs.acquire) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + threads = [threading.Thread(target=bs.release) + for _ in range(limit)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertRaises(ValueError, bs.release) class ThreadJoinOnShutdown(BaseTestCase): diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index e44fe03d240..666cab87255 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -1658,6 +1658,18 @@ def test_encode_decimal(self): self.assertEqual(unicode_encodedecimal(u"123\u20ac\u0660", "replace"), b'123?0') + def test_encode_decimal_with_surrogates(self): + from _testcapi import unicode_encodedecimal + tests = [(u'\U0001f49d', '💝'), + (u'\ud83d', '�'), + (u'\udc9d', '�'), + (u'\ud83d\udc9d', '💝' if len(u'\U0001f49d') > 1 else + '��'), + ] + for s, exp in tests: + self.assertEqual( + unicode_encodedecimal(u"123" + s, "xmlcharrefreplace"), + '123' + exp) def test_main(): test_support.run_unittest(__name__) diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 3a273f873b1..3018ab198b4 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -227,13 +227,13 @@ def test_missing_localfile(self): 'file://localhost/a/missing/file.py') fd, tmp_file = tempfile.mkstemp() tmp_fileurl = 'file://localhost/' + tmp_file.replace(os.path.sep, '/') + self.assertTrue(os.path.exists(tmp_file)) try: - self.assertTrue(os.path.exists(tmp_file)) fp = urllib.urlopen(tmp_fileurl) + fp.close() finally: os.close(fd) - fp.close() - os.unlink(tmp_file) + os.unlink(tmp_file) self.assertFalse(os.path.exists(tmp_file)) self.assertRaises(IOError, urllib.urlopen, tmp_fileurl) @@ -812,7 +812,7 @@ def open_spam(self, url): # Everywhere else they work ok, but on those machines, sometimes # fail in one of the tests, sometimes in other. I have a linux, and # the tests go ok. -# If anybody has one of the problematic enviroments, please help! +# If anybody has one of the problematic environments, please help! # . Facundo # # def server(evt): diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index e5030849184..9de3d789c5e 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -1,5 +1,6 @@ -from unittest import TestCase +import unittest from test import test_support +import os import uuid def importable(name): @@ -9,7 +10,7 @@ def importable(name): except: return False -class TestUUID(TestCase): +class TestUUID(unittest.TestCase): last_node = None source2node = {} @@ -299,24 +300,22 @@ def check_node(self, node, source): else: TestUUID.last_node = node + @unittest.skipUnless(os.name == 'posix', 'requires Posix') def test_ifconfig_getnode(self): - import sys - import os - if os.name == 'posix': - node = uuid._ifconfig_getnode() - if node is not None: - self.check_node(node, 'ifconfig') + node = uuid._ifconfig_getnode() + if node is not None: + self.check_node(node, 'ifconfig') + @unittest.skipUnless(os.name == 'nt', 'requires Windows') def test_ipconfig_getnode(self): - import os - if os.name == 'nt': - node = uuid._ipconfig_getnode() - if node is not None: - self.check_node(node, 'ipconfig') + node = uuid._ipconfig_getnode() + if node is not None: + self.check_node(node, 'ipconfig') + @unittest.skipUnless(importable('win32wnet'), 'requires win32wnet') + @unittest.skipUnless(importable('netbios'), 'requires netbios') def test_netbios_getnode(self): - if importable('win32wnet') and importable('netbios'): - self.check_node(uuid._netbios_getnode(), 'netbios') + self.check_node(uuid._netbios_getnode(), 'netbios') def test_random_getnode(self): node = uuid._random_getnode() @@ -324,22 +323,20 @@ def test_random_getnode(self): self.assertTrue(node & 0x010000000000) self.assertTrue(node < (1L << 48)) + @unittest.skipUnless(os.name == 'posix', 'requires Posix') + @unittest.skipUnless(importable('ctypes'), 'requires ctypes') def test_unixdll_getnode(self): - import sys - import os - if importable('ctypes') and os.name == 'posix': - try: # Issues 1481, 3581: _uuid_generate_time() might be None. - self.check_node(uuid._unixdll_getnode(), 'unixdll') - except TypeError: - pass + try: # Issues 1481, 3581: _uuid_generate_time() might be None. + self.check_node(uuid._unixdll_getnode(), 'unixdll') + except TypeError: + pass + @unittest.skipUnless(os.name == 'nt', 'requires Windows') + @unittest.skipUnless(importable('ctypes'), 'requires ctypes') def test_windll_getnode(self): - import os - if importable('ctypes') and os.name == 'nt': - self.check_node(uuid._windll_getnode(), 'windll') + self.check_node(uuid._windll_getnode(), 'windll') def test_getnode(self): - import sys node1 = uuid.getnode() self.check_node(node1, "getnode1") @@ -349,13 +346,8 @@ def test_getnode(self): self.assertEqual(node1, node2) + @unittest.skipUnless(importable('ctypes'), 'requires ctypes') def test_uuid1(self): - # uuid1 requires ctypes. - try: - import ctypes - except ImportError: - return - equal = self.assertEqual # Make sure uuid1() generates UUIDs that are actually version 1. @@ -408,13 +400,8 @@ def test_uuid3(self): equal(u, uuid.UUID(v)) equal(str(u), v) + @unittest.skipUnless(importable('ctypes'), 'requires ctypes') def test_uuid4(self): - # uuid4 requires ctypes. - try: - import ctypes - except ImportError: - return - equal = self.assertEqual # Make sure uuid4() generates UUIDs that are actually version 4. @@ -446,12 +433,8 @@ def test_uuid5(self): equal(u, uuid.UUID(v)) equal(str(u), v) + @unittest.skipUnless(os.name == 'posix', 'requires Posix') def testIssue8621(self): - import os - import sys - if os.name != 'posix': - return - # On at least some versions of OSX uuid.uuid4 generates # the same sequence of UUIDs in the parent and any # children started using fork. @@ -465,6 +448,7 @@ def testIssue8621(self): else: os.close(fds[1]) + self.addCleanup(os.close, fds[0]) parent_value = uuid.uuid4().hex os.waitpid(pid, 0) child_value = os.read(fds[0], 100) diff --git a/Lib/test/test_wait4.py b/Lib/test/test_wait4.py index d04a11bde10..54580a90266 100644 --- a/Lib/test/test_wait4.py +++ b/Lib/test/test_wait4.py @@ -3,6 +3,7 @@ import os import time +import sys from test.fork_wait import ForkWait from test.test_support import run_unittest, reap_children, get_attribute @@ -13,10 +14,15 @@ class Wait4Test(ForkWait): def wait_impl(self, cpid): + option = os.WNOHANG + if sys.platform.startswith('aix'): + # Issue #11185: wait4 is broken on AIX and will always return 0 + # with WNOHANG. + option = 0 for i in range(10): # wait4() shouldn't hang, but some of the buildbots seem to hang # in the forking tests. This is an attempt to fix the problem. - spid, status, rusage = os.wait4(cpid, os.WNOHANG) + spid, status, rusage = os.wait4(cpid, option) if spid == cpid: break time.sleep(1.0) diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py index d9e091ed02f..fb9e8d73db3 100644 --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -351,6 +351,12 @@ def test_eq(self): self.assertFalse(self.s == tuple(self.items)) self.assertFalse(self.s == 1) + def test_ne(self): + self.assertTrue(self.s != set(self.items)) + s1 = WeakSet() + s2 = WeakSet() + self.assertFalse(s1 != s2) + def test_weak_destroy_while_iterating(self): # Issue #7105: iterators shouldn't crash when a key is implicitly removed # Create new items to be sure no-one else holds a reference diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py index 2799e194802..e05d040998b 100644 --- a/Lib/test/test_winreg.py +++ b/Lib/test/test_winreg.py @@ -28,9 +28,12 @@ # tests are only valid up until 6.1 HAS_REFLECTION = True if WIN_VER < (6, 1) else False -test_key_name = "SOFTWARE\\Python Registry Test Key - Delete Me" +# Use a per-process key to prevent concurrent test runs (buildbot!) from +# stomping on each other. +test_key_base = "Python Test Key [%d] - Delete Me" % (os.getpid(),) +test_key_name = "SOFTWARE\\" + test_key_base # On OS'es that support reflection we should test with a reflected key -test_reflect_key_name = "SOFTWARE\\Classes\\Python Test Key - Delete Me" +test_reflect_key_name = "SOFTWARE\\Classes\\" + test_key_base test_data = [ ("Int Value", 45, REG_DWORD), @@ -439,6 +442,11 @@ def test_disable_reflection(self): DeleteKeyEx(HKEY_CURRENT_USER, test_reflect_key_name, KEY_WOW64_32KEY, 0) + def test_exception_numbers(self): + with self.assertRaises(WindowsError) as ctx: + QueryValue(HKEY_CLASSES_ROOT, 'some_value_that_does_not_exist') + + self.assertEqual(ctx.exception.errno, 2) def test_main(): test_support.run_unittest(LocalWinregTests, RemoteWinregTests, diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 4e161ca2f3a..adb785247d5 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -883,6 +883,12 @@ def check_encoding(encoding): >>> check_encoding("iso-8859-15") >>> check_encoding("cp437") >>> check_encoding("mac-roman") + >>> check_encoding("gbk") + Traceback (most recent call last): + ValueError: multi-byte encodings are not supported + >>> check_encoding("cp037") + Traceback (most recent call last): + ParseError: unknown encoding: line 1, column 30 """ ET.XML("" % encoding) @@ -1769,6 +1775,16 @@ def bug_200709_iter_comment(): """ +def bug_18347(): + """ + + >>> e = ET.XML('text') + >>> serialize(e) + 'text' + >>> serialize(e, method="html") + 'text' + """ + # -------------------------------------------------------------------- # reported on bugs.python.org diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index c26ca80b0a9..9e3a28b4383 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -18,7 +18,14 @@ from random import randint, random from unittest import skipUnless -from test.test_support import TESTFN, run_unittest, findfile, unlink +from test.test_support import TESTFN, TESTFN_UNICODE, TESTFN_ENCODING, \ + run_unittest, findfile, unlink +try: + TESTFN_UNICODE.encode(TESTFN_ENCODING) +except (UnicodeError, TypeError): + # Either the file system encoding is None, or the file name + # cannot be encoded in the file system encoding. + TESTFN_UNICODE = None TESTFN2 = TESTFN + "2" TESTFNDIR = TESTFN + "d" @@ -424,6 +431,26 @@ def check_file(self, filename, content): with open(filename, 'rb') as f: self.assertEqual(f.read(), content) + @skipUnless(TESTFN_UNICODE, "No Unicode filesystem semantics on this platform.") + def test_extract_unicode_filenames(self): + fnames = [u'foo.txt', os.path.basename(TESTFN_UNICODE)] + content = 'Test for unicode filename' + with zipfile.ZipFile(TESTFN2, "w", zipfile.ZIP_STORED) as zipfp: + for fname in fnames: + zipfp.writestr(fname, content) + + with zipfile.ZipFile(TESTFN2, "r") as zipfp: + for fname in fnames: + writtenfile = zipfp.extract(fname) + + # make sure it was written to the right place + correctfile = os.path.join(os.getcwd(), fname) + correctfile = os.path.normpath(correctfile) + self.assertEqual(writtenfile, correctfile) + + self.check_file(writtenfile, content) + os.remove(writtenfile) + def test_extract_hackers_arcnames(self): hacknames = [ ('../foo/bar', 'foo/bar'), diff --git a/Lib/test/testbz2_bigmem.bz2 b/Lib/test/testbz2_bigmem.bz2 deleted file mode 100644 index c9a4616c805..00000000000 Binary files a/Lib/test/testbz2_bigmem.bz2 and /dev/null differ diff --git a/Lib/threading.py b/Lib/threading.py index cb49c4a7ff8..e81471bbb00 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -457,21 +457,20 @@ def acquire(self, blocking=1): """ rc = False - self.__cond.acquire() - while self.__value == 0: - if not blocking: - break - if __debug__: - self._note("%s.acquire(%s): blocked waiting, value=%s", - self, blocking, self.__value) - self.__cond.wait() - else: - self.__value = self.__value - 1 - if __debug__: - self._note("%s.acquire: success, value=%s", - self, self.__value) - rc = True - self.__cond.release() + with self.__cond: + while self.__value == 0: + if not blocking: + break + if __debug__: + self._note("%s.acquire(%s): blocked waiting, value=%s", + self, blocking, self.__value) + self.__cond.wait() + else: + self.__value = self.__value - 1 + if __debug__: + self._note("%s.acquire: success, value=%s", + self, self.__value) + rc = True return rc __enter__ = acquire @@ -483,13 +482,12 @@ def release(self): to become larger than zero again, wake up that thread. """ - self.__cond.acquire() - self.__value = self.__value + 1 - if __debug__: - self._note("%s.release: success, value=%s", - self, self.__value) - self.__cond.notify() - self.__cond.release() + with self.__cond: + self.__value = self.__value + 1 + if __debug__: + self._note("%s.release: success, value=%s", + self, self.__value) + self.__cond.notify() def __exit__(self, t, v, tb): self.release() @@ -533,9 +531,11 @@ def release(self): raise a ValueError. """ - if self._Semaphore__value >= self._initial_value: - raise ValueError("Semaphore released too many times") - return _Semaphore.release(self) + with self._Semaphore__cond: + if self._Semaphore__value >= self._initial_value: + raise ValueError("Semaphore released too many times") + self._Semaphore__value += 1 + self._Semaphore__cond.notify() def Event(*args, **kwargs): @@ -1222,7 +1222,7 @@ def _after_fork(): new_active = {} current = current_thread() with _active_limbo_lock: - for thread in _active.itervalues(): + for thread in _enumerate(): # Any lock/condition variable may be currently locked or in an # invalid state, so we reinitialize them. if hasattr(thread, '_reset_internal_locks'): diff --git a/Lib/unittest/__init__.py b/Lib/unittest/__init__.py index 201a3f0d2da..a5d50af78f2 100644 --- a/Lib/unittest/__init__.py +++ b/Lib/unittest/__init__.py @@ -11,7 +11,7 @@ import unittest - class IntegerArithmenticTestCase(unittest.TestCase): + class IntegerArithmeticTestCase(unittest.TestCase): def testAdd(self): ## test method names begin 'test*' self.assertEqual((1 + 2), 3) self.assertEqual(0 + 1, 1) diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index e88f536b394..c6fc6630de1 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -106,7 +106,9 @@ def loadTestsFromName(self, name, module=None): elif (isinstance(obj, types.UnboundMethodType) and isinstance(parent, type) and issubclass(parent, case.TestCase)): - return self.suiteClass([parent(obj.__name__)]) + name = parts[-1] + inst = parent(name) + return self.suiteClass([inst]) elif isinstance(obj, suite.TestSuite): return obj elif hasattr(obj, '__call__'): diff --git a/Lib/unittest/test/test_assertions.py b/Lib/unittest/test/test_assertions.py index e1ba6147066..e8f0f64b4cf 100644 --- a/Lib/unittest/test/test_assertions.py +++ b/Lib/unittest/test/test_assertions.py @@ -33,6 +33,10 @@ def test_AmostEqualWithDelta(self): self.assertNotAlmostEqual(1.1, 1.0, delta=0.05) self.assertNotAlmostEqual(1.0, 1.1, delta=0.05) + self.assertAlmostEqual(1.0, 1.0, delta=0.5) + self.assertRaises(self.failureException, self.assertNotAlmostEqual, + 1.0, 1.0, delta=0.5) + self.assertRaises(self.failureException, self.assertAlmostEqual, 1.1, 1.0, delta=0.05) self.assertRaises(self.failureException, self.assertNotAlmostEqual, diff --git a/Lib/unittest/test/test_loader.py b/Lib/unittest/test/test_loader.py index 3544a20f7cf..3e9756e65ae 100644 --- a/Lib/unittest/test/test_loader.py +++ b/Lib/unittest/test/test_loader.py @@ -324,7 +324,7 @@ def test_loadTestsFromName__relative_malformed_name(self): # Does loadTestsFromName raise TypeError when the `module` argument # isn't a module object? # - # XXX Accepts the not-a-module object, ignorning the object's type + # XXX Accepts the not-a-module object, ignoring the object's type # This should raise an exception or the method name should be changed # # XXX Some people are relying on this, so keep it for now @@ -1281,6 +1281,21 @@ def test_suiteClass__default_value(self): loader = unittest.TestLoader() self.assertTrue(loader.suiteClass is unittest.TestSuite) + # Make sure the dotted name resolution works even if the actual + # function doesn't have the same name as is used to find it. + def test_loadTestsFromName__function_with_different_name_than_method(self): + # lambdas have the name ''. + m = types.ModuleType('m') + class MyTestCase(unittest.TestCase): + test = lambda: 1 + m.testcase_1 = MyTestCase + + loader = unittest.TestLoader() + suite = loader.loadTestsFromNames(['testcase_1.test'], m) + self.assertIsInstance(suite, loader.suiteClass) + + ref_suite = unittest.TestSuite([MyTestCase('test')]) + self.assertEqual(list(suite), [ref_suite]) if __name__ == '__main__': unittest.main() diff --git a/Lib/unittest/test/test_runner.py b/Lib/unittest/test/test_runner.py index d1cefae4c17..3b661d33408 100644 --- a/Lib/unittest/test/test_runner.py +++ b/Lib/unittest/test/test_runner.py @@ -159,7 +159,7 @@ class ATextResult(unittest.TextTestResult, AResult): # This used to raise an exception due to TextTestResult not passing # on arguments in its __init__ super call - ATextResult(None, None, None) + ATextResult(None, None, 1) def testBufferAndFailfast(self): diff --git a/Lib/urllib.py b/Lib/urllib.py index f9655f9e88a..244cb05374d 100644 --- a/Lib/urllib.py +++ b/Lib/urllib.py @@ -819,7 +819,10 @@ def thishost(): """Return the IP address of the current host.""" global _thishost if _thishost is None: - _thishost = socket.gethostbyname(socket.gethostname()) + try: + _thishost = socket.gethostbyname(socket.gethostname()) + except socket.gaierror: + _thishost = socket.gethostbyname('localhost') return _thishost _ftperrors = None @@ -870,8 +873,8 @@ def init(self): self.ftp = ftplib.FTP() self.ftp.connect(self.host, self.port, self.timeout) self.ftp.login(self.user, self.passwd) - for dir in self.dirs: - self.ftp.cwd(dir) + _target = '/'.join(self.dirs) + self.ftp.cwd(_target) def retrfile(self, file, type): import ftplib diff --git a/Lib/uuid.py b/Lib/uuid.py index fdd0c5cbec0..cb4e5f050f1 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -408,6 +408,8 @@ def _netbios_getnode(): _uuid_generate_random = lib.uuid_generate_random if hasattr(lib, 'uuid_generate_time'): _uuid_generate_time = lib.uuid_generate_time + if _uuid_generate_random is not None: + break # found everything we were looking for # The uuid_generate_* functions are broken on MacOS X 10.5, as noted # in issue #8621 the function generates the same sequence of values diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index f3c53d423cf..d3272a0a48e 100644 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -304,6 +304,18 @@ class Galeon(UnixBrowser): background = True +class Chrome(UnixBrowser): + "Launcher class for Google Chrome browser." + + remote_args = ['%action', '%s'] + remote_action = "" + remote_action_newwin = "--new-window" + remote_action_newtab = "" + background = True + +Chromium = Chrome + + class Opera(UnixBrowser): "Launcher class for Opera browser." @@ -441,6 +453,14 @@ def open(self, url, new=0, autoraise=True): def register_X_browsers(): + # use xdg-open if around + if _iscommand("xdg-open"): + register("xdg-open", None, BackgroundBrowser("xdg-open")) + + # The default GNOME3 browser + if "GNOME_DESKTOP_SESSION_ID" in os.environ and _iscommand("gvfs-open"): + register("gvfs-open", None, BackgroundBrowser("gvfs-open")) + # The default GNOME browser if "GNOME_DESKTOP_SESSION_ID" in os.environ and _iscommand("gnome-open"): register("gnome-open", None, BackgroundBrowser("gnome-open")) @@ -449,9 +469,13 @@ def register_X_browsers(): if "KDE_FULL_SESSION" in os.environ and _iscommand("kfmclient"): register("kfmclient", Konqueror, Konqueror("kfmclient")) + if _iscommand("x-www-browser"): + register("x-www-browser", None, BackgroundBrowser("x-www-browser")) + # The Mozilla/Netscape browsers for browser in ("mozilla-firefox", "firefox", "mozilla-firebird", "firebird", + "iceweasel", "iceape", "seamonkey", "mozilla", "netscape"): if _iscommand(browser): register(browser, None, Mozilla(browser)) @@ -471,6 +495,11 @@ def register_X_browsers(): if _iscommand("skipstone"): register("skipstone", None, BackgroundBrowser("skipstone")) + # Google Chrome/Chromium browsers + for browser in ("google-chrome", "chrome", "chromium", "chromium-browser"): + if _iscommand(browser): + register(browser, None, Chrome(browser)) + # Opera, quite popular if _iscommand("opera"): register("opera", None, Opera("opera")) @@ -489,6 +518,8 @@ def register_X_browsers(): # Also try console browsers if os.environ.get("TERM"): + if _iscommand("www-browser"): + register("www-browser", None, GenericBrowser("www-browser")) # The Links/elinks browsers if _iscommand("links"): register("links", None, GenericBrowser("links")) diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index f8ed232fb2e..c30e2462ee9 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -358,9 +358,6 @@ def __init__(self, qName, namespaceURI=EMPTY_NAMESPACE, localName=None, def _get_localName(self): return self.nodeName.split(":", 1)[-1] - def _get_name(self): - return self.name - def _get_specified(self): return self.specified diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index bb468cde698..9f3e75d4a6f 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -988,15 +988,15 @@ def _serialize_html(write, elem, encoding, qnames, namespaces): # FIXME: handle boolean attributes write(" %s=\"%s\"" % (qnames[k], v)) write(">") - tag = tag.lower() + ltag = tag.lower() if text: - if tag == "script" or tag == "style": + if ltag == "script" or ltag == "style": write(_encode(text, encoding)) else: write(_escape_cdata(text, encoding)) for e in elem: _serialize_html(write, e, encoding, qnames, None) - if tag not in HTML_EMPTY: + if ltag not in HTML_EMPTY: write("") if elem.tail: write(_escape_cdata(elem.tail, encoding)) diff --git a/Lib/xml/sax/saxutils.py b/Lib/xml/sax/saxutils.py index 3d81a8e1bf4..1abcd9a0c4a 100644 --- a/Lib/xml/sax/saxutils.py +++ b/Lib/xml/sax/saxutils.py @@ -180,10 +180,14 @@ def endElementNS(self, name, qname): self._write(u'' % self._qname(name)) def characters(self, content): - self._write(escape(unicode(content))) + if not isinstance(content, unicode): + content = unicode(content, self._encoding) + self._write(escape(content)) def ignorableWhitespace(self, content): - self._write(unicode(content)) + if not isinstance(content, unicode): + content = unicode(content, self._encoding) + self._write(content) def processingInstruction(self, target, data): self._write(u'' % (target, data)) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 9d1a98479db..6639317537b 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -1053,7 +1053,10 @@ def _extract_member(self, member, targetpath, pwd): if os.path.sep == '\\': # filter illegal characters on Windows illegal = ':<>|"?*' - table = string.maketrans(illegal, '_' * len(illegal)) + if isinstance(arcname, unicode): + table = {ord(c): ord('_') for c in illegal} + else: + table = string.maketrans(illegal, '_' * len(illegal)) arcname = arcname.translate(table) # remove trailing dots arcname = (x.rstrip('.') for x in arcname.split(os.path.sep)) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 004765ae580..ef822d58549 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -108,7 +108,7 @@ def getFullVersion(): ### There are some issues with the SDK selection below here, ### The resulting binary doesn't work on all platforms that ### it should. Always default to the 10.4u SDK until that -### isue is resolved. +### issue is resolved. ### ##if int(os.uname()[2].split('.')[0]) == 8: ## # Explicitly use the 10.4u (universal) SDK when @@ -1015,7 +1015,7 @@ def buildPython(): os.chdir(curdir) if PYTHON_3: - # Remove the 'Current' link, that way we don't accidently mess + # Remove the 'Current' link, that way we don't accidentally mess # with an already installed version of python 2 os.unlink(os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework', 'Versions', 'Current')) diff --git a/Mac/Makefile.in b/Mac/Makefile.in index 6e60b2af344..a3e00b22f91 100644 --- a/Mac/Makefile.in +++ b/Mac/Makefile.in @@ -202,15 +202,22 @@ install_IDLE: cd IDLE && make install install_BuildApplet: - $(RUNSHARED) @ARCH_RUN_32BIT@ $(BUILDPYTHON) $(srcdir)/scripts/BuildApplet.py \ - --destroot "$(DESTDIR)" \ - --python=$(prefix)/Resources/Python.app/Contents/MacOS/Python \ - --output "$(DESTDIR)$(PYTHONAPPSDIR)/Build Applet.app" \ - $(srcdir)/scripts/BuildApplet.py -ifneq ($(LIPO_32BIT_FLAGS),) - rm "$(DESTDIR)$(PYTHONAPPSDIR)/Build Applet.app/Contents/MacOS/Python" - lipo $(LIPO_32BIT_FLAGS) -output "$(DESTDIR)$(PYTHONAPPSDIR)/Build Applet.app/Contents/MacOS/Python" $(BUILDPYTHON) -endif + if ! $(RUNSHARED) @ARCH_RUN_32BIT@ $(BUILDPYTHON) \ + -c 'import EasyDialogs' 2>/dev/null ; then \ + echo "EasyDialogs not available in this Python - skipping Build Applet.app" ; \ + else \ + $(RUNSHARED) @ARCH_RUN_32BIT@ $(BUILDPYTHON) $(srcdir)/scripts/BuildApplet.py \ + --destroot "$(DESTDIR)" \ + --python=$(prefix)/Resources/Python.app/Contents/MacOS/Python \ + --output "$(DESTDIR)$(PYTHONAPPSDIR)/Build Applet.app" \ + $(srcdir)/scripts/BuildApplet.py && \ + if [ -n "$(LIPO_32BIT_FLAGS)" ] ; then \ + rm "$(DESTDIR)$(PYTHONAPPSDIR)/Build Applet.app/Contents/MacOS/Python" && \ + lipo $(LIPO_32BIT_FLAGS) \ + -output "$(DESTDIR)$(PYTHONAPPSDIR)/Build Applet.app/Contents/MacOS/Python" \ + $(BUILDPYTHON) ; \ + fi \ + fi MACLIBDEST=$(LIBDEST)/plat-mac MACTOOLSDEST=$(prefix)/Mac/Tools diff --git a/Mac/PythonLauncher/FileSettings.h b/Mac/PythonLauncher/FileSettings.h index d807bae59cc..7b74a9b24d2 100755 --- a/Mac/PythonLauncher/FileSettings.h +++ b/Mac/PythonLauncher/FileSettings.h @@ -45,18 +45,13 @@ + (id)getFactorySettingsForFileType: (NSString *)filetype; + (id)newSettingsForFileType: (NSString *)filetype; -//- (id)init; - (id)initForFileType: (NSString *)filetype; - (id)initForFSDefaultFileType: (NSString *)filetype; - (id)initForDefaultFileType: (NSString *)filetype; -//- (id)initWithFileSettings: (FileSettings *)source; - (void)updateFromSource: (id )source; - (NSString *)commandLineForScript: (NSString *)script; -//- (void)applyFactorySettingsForFileType: (NSString *)filetype; -//- (void)saveDefaults; -//- (void)applyUserDefaults: (NSString *)filetype; - (void)applyValuesFromDict: (NSDictionary *)dict; - (void)reset; - (NSArray *) interpreters; diff --git a/Mac/PythonLauncher/FileSettings.m b/Mac/PythonLauncher/FileSettings.m index 66b4fdc0866..3438870b748 100755 --- a/Mac/PythonLauncher/FileSettings.m +++ b/Mac/PythonLauncher/FileSettings.m @@ -14,7 +14,7 @@ + (id)getFactorySettingsForFileType: (NSString *)filetype { static FileSettings *fsdefault_py, *fsdefault_pyw, *fsdefault_pyc; FileSettings **curdefault; - + if ([filetype isEqualToString: @"Python Script"]) { curdefault = &fsdefault_py; } else if ([filetype isEqualToString: @"Python GUI Script"]) { @@ -36,7 +36,7 @@ + (id)getDefaultsForFileType: (NSString *)filetype { static FileSettings *default_py, *default_pyw, *default_pyc; FileSettings **curdefault; - + if ([filetype isEqualToString: @"Python Script"]) { curdefault = &default_py; } else if ([filetype isEqualToString: @"Python GUI Script"]) { @@ -57,7 +57,7 @@ + (id)getDefaultsForFileType: (NSString *)filetype + (id)newSettingsForFileType: (NSString *)filetype { FileSettings *cur; - + cur = [FileSettings new]; [cur initForFileType: filetype]; return [cur retain]; @@ -67,7 +67,7 @@ - (id)initWithFileSettings: (FileSettings *)source { self = [super init]; if (!self) return self; - + interpreter = [source->interpreter retain]; honourhashbang = source->honourhashbang; debug = source->debug; @@ -81,36 +81,30 @@ - (id)initWithFileSettings: (FileSettings *)source with_terminal = source->with_terminal; prefskey = source->prefskey; if (prefskey) [prefskey retain]; - + return self; } - (id)initForFileType: (NSString *)filetype { FileSettings *defaults; - + defaults = [FileSettings getDefaultsForFileType: filetype]; self = [self initWithFileSettings: defaults]; origsource = [defaults retain]; return self; } -//- (id)init -//{ -// self = [self initForFileType: @"Python Script"]; -// return self; -//} - - (id)initForFSDefaultFileType: (NSString *)filetype { int i; NSString *filename; NSDictionary *dict; static NSDictionary *factorySettings; - + self = [super init]; if (!self) return self; - + if (factorySettings == NULL) { NSBundle *bdl = [NSBundle mainBundle]; NSString *path = [ bdl pathForResource: @"factorySettings" @@ -149,18 +143,18 @@ - (void)applyUserDefaults: (NSString *)filetype { NSUserDefaults *defaults; NSDictionary *dict; - + defaults = [NSUserDefaults standardUserDefaults]; dict = [defaults dictionaryForKey: filetype]; if (!dict) return; [self applyValuesFromDict: dict]; } - + - (id)initForDefaultFileType: (NSString *)filetype { FileSettings *fsdefaults; - + fsdefaults = [FileSettings getFactorySettingsForFileType: filetype]; self = [self initWithFileSettings: fsdefaults]; if (!self) return self; @@ -220,7 +214,7 @@ - (void)updateFromSource: (id )source - (void)applyValuesFromDict: (NSDictionary *)dict { id value; - + value = [dict objectForKey: @"interpreter"]; if (value) interpreter = [value retain]; value = [dict objectForKey: @"honourhashbang"]; @@ -247,12 +241,12 @@ - (void)applyValuesFromDict: (NSDictionary *)dict - (NSString*)_replaceSingleQuotes: (NSString*)string { - /* Replace all single-quotes by '"'"', that way shellquoting will - * be correct when the result value is delimited using single quotes. - */ - NSArray* components = [string componentsSeparatedByString:@"'"]; + /* Replace all single-quotes by '"'"', that way shellquoting will + * be correct when the result value is delimited using single quotes. + */ + NSArray* components = [string componentsSeparatedByString:@"'"]; - return [components componentsJoinedByString:@"'\"'\"'"]; + return [components componentsJoinedByString:@"'\"'\"'"]; } - (NSString *)commandLineForScript: (NSString *)script @@ -265,7 +259,7 @@ - (NSString *)commandLineForScript: (NSString *)script script_dir = [script substringToIndex: [script length]-[[script lastPathComponent] length]]; - + if (honourhashbang && (fp=fopen([script fileSystemRepresentation], "r")) && fgets(hashbangbuf, sizeof(hashbangbuf), fp) && @@ -278,7 +272,7 @@ - (NSString *)commandLineForScript: (NSString *)script } if (!cur_interp) cur_interp = interpreter; - + return [NSString stringWithFormat: @"cd '%@' && '%@'%s%s%s%s%s%s %@ '%@' %@ %s", [self _replaceSingleQuotes:script_dir], @@ -297,7 +291,7 @@ - (NSString *)commandLineForScript: (NSString *)script - (NSArray *) interpreters { return interpreters;}; -// FileSettingsSource protocol +// FileSettingsSource protocol - (NSString *) interpreter { return interpreter;}; - (BOOL) honourhashbang { return honourhashbang; }; - (BOOL) debug { return debug;}; diff --git a/Mac/PythonLauncher/MyAppDelegate.m b/Mac/PythonLauncher/MyAppDelegate.m index a5ba7510784..e75fb06616b 100644 --- a/Mac/PythonLauncher/MyAppDelegate.m +++ b/Mac/PythonLauncher/MyAppDelegate.m @@ -33,7 +33,7 @@ - (void)applicationDidFinishLaunching:(NSNotification *)notification - (BOOL)shouldShowUI { - // if this call comes before applicationDidFinishLaunching: we + // if this call comes before applicationDidFinishLaunching: we // should terminate immedeately after starting the script. if (!initial_action_done) should_terminate = YES; @@ -62,7 +62,7 @@ - (void)testFileTypeBinding static NSString *extensions[] = { @"py", @"pyw", @"pyc", NULL}; NSString **ext_p; int i; - + if ([[NSUserDefaults standardUserDefaults] boolForKey: @"SkipFileBindingTest"]) return; ourUrl = [NSURL fileURLWithPath: [[NSBundle mainBundle] bundlePath]]; @@ -92,5 +92,5 @@ - (void)testFileTypeBinding } } } - + @end diff --git a/Mac/PythonLauncher/MyDocument.m b/Mac/PythonLauncher/MyDocument.m index 86112c4b15d..90c5db9dc5b 100755 --- a/Mac/PythonLauncher/MyDocument.m +++ b/Mac/PythonLauncher/MyDocument.m @@ -16,7 +16,7 @@ - (id)init { self = [super init]; if (self) { - + // Add your subclass-specific initialization here. // If an error occurs here, send a [self dealloc] message and return nil. script = [@".py" retain]; @@ -37,20 +37,17 @@ - (void)close { NSApplication *app = [NSApplication sharedApplication]; [super close]; - if ([[app delegate] shouldTerminate]) + if ([(MyAppDelegate*)[app delegate] shouldTerminate]) [app terminate: self]; } - (void)load_defaults { -// if (settings) [settings release]; settings = [FileSettings newSettingsForFileType: filetype]; } - (void)update_display { -// [[self window] setTitle: script]; - [interpreter setStringValue: [settings interpreter]]; [honourhashbang setState: [settings honourhashbang]]; [debug setState: [settings debug]]; @@ -62,7 +59,7 @@ - (void)update_display [others setStringValue: [settings others]]; [scriptargs setStringValue: [settings scriptargs]]; [with_terminal setState: [settings with_terminal]]; - + [commandline setStringValue: [settings commandLineForScript: script]]; } @@ -75,8 +72,8 @@ - (BOOL)run { const char *cmdline; int sts; - - cmdline = [[settings commandLineForScript: script] cString]; + + cmdline = [[settings commandLineForScript: script] UTF8String]; if ([settings with_terminal]) { sts = doscript(cmdline); } else { @@ -107,14 +104,13 @@ - (BOOL)readFromFile:(NSString *)fileName ofType:(NSString *)type; { // Insert code here to read your document from the given data. You can also choose to override -loadFileWrapperRepresentation:ofType: or -readFromFile:ofType: instead. BOOL show_ui; - - // ask the app delegate whether we should show the UI or not. - show_ui = [[[NSApplication sharedApplication] delegate] shouldShowUI]; + + // ask the app delegate whether we should show the UI or not. + show_ui = [(MyAppDelegate*)[[NSApplication sharedApplication] delegate] shouldShowUI]; [script release]; script = [fileName retain]; [filetype release]; filetype = [type retain]; -// if (settings) [settings release]; settings = [FileSettings newSettingsForFileType: filetype]; if (show_ui) { [self update_display]; @@ -152,7 +148,7 @@ - (IBAction)do_apply:(id)sender [self update_display]; } -// FileSettingsSource protocol +// FileSettingsSource protocol - (NSString *) interpreter { return [interpreter stringValue];}; - (BOOL) honourhashbang { return [honourhashbang state];}; - (BOOL) debug { return [debug state];}; diff --git a/Mac/PythonLauncher/PreferencesWindowController.m b/Mac/PythonLauncher/PreferencesWindowController.m index 311c37502bb..ec5bbe84c8c 100644 --- a/Mac/PythonLauncher/PreferencesWindowController.m +++ b/Mac/PythonLauncher/PreferencesWindowController.m @@ -5,7 +5,7 @@ @implementation PreferencesWindowController + getPreferencesWindow { static PreferencesWindowController *_singleton; - + if (!_singleton) _singleton = [[PreferencesWindowController alloc] init]; [_singleton showWindow: _singleton]; @@ -21,15 +21,13 @@ - (id) init - (void)load_defaults { NSString *title = [filetype titleOfSelectedItem]; - + settings = [FileSettings getDefaultsForFileType: title]; } - (void)update_display { -// [[self window] setTitle: script]; - - [interpreter reloadData]; + [interpreter reloadData]; [interpreter setStringValue: [settings interpreter]]; [honourhashbang setState: [settings honourhashbang]]; [debug setState: [settings debug]]; @@ -41,7 +39,6 @@ - (void)update_display [others setStringValue: [settings others]]; [with_terminal setState: [settings with_terminal]]; // Not scriptargs, it isn't for preferences - [commandline setStringValue: [settings commandLineForScript: @""]]; } @@ -75,7 +72,7 @@ - (IBAction)do_apply:(id)sender [self update_display]; } -// FileSettingsSource protocol +// FileSettingsSource protocol - (NSString *) interpreter { return [interpreter stringValue];}; - (BOOL) honourhashbang { return [honourhashbang state]; }; - (BOOL) debug { return [debug state];}; @@ -98,23 +95,23 @@ - (void)controlTextDidChange:(NSNotification *)aNotification // NSComboBoxDataSource protocol - (unsigned int)comboBox:(NSComboBox *)aComboBox indexOfItemWithStringValue:(NSString *)aString { - NSArray *interp_list = [settings interpreters]; + NSArray *interp_list = [settings interpreters]; unsigned int rv = [interp_list indexOfObjectIdenticalTo: aString]; - return rv; + return rv; } - (id)comboBox:(NSComboBox *)aComboBox objectValueForItemAtIndex:(int)index { - NSArray *interp_list = [settings interpreters]; + NSArray *interp_list = [settings interpreters]; id rv = [interp_list objectAtIndex: index]; - return rv; + return rv; } - (int)numberOfItemsInComboBox:(NSComboBox *)aComboBox { - NSArray *interp_list = [settings interpreters]; + NSArray *interp_list = [settings interpreters]; int rv = [interp_list count]; - return rv; + return rv; } diff --git a/Mac/PythonLauncher/doscript.h b/Mac/PythonLauncher/doscript.h index eef0b5641f6..3fd3187cd4a 100644 --- a/Mac/PythonLauncher/doscript.h +++ b/Mac/PythonLauncher/doscript.h @@ -9,4 +9,4 @@ #include -extern int doscript(const char *command); \ No newline at end of file +extern int doscript(const char *command); diff --git a/Mac/PythonLauncher/doscript.m b/Mac/PythonLauncher/doscript.m index 024b883f190..cbb783ba3e8 100644 --- a/Mac/PythonLauncher/doscript.m +++ b/Mac/PythonLauncher/doscript.m @@ -11,49 +11,49 @@ #import #import "doscript.h" -extern int +extern int doscript(const char *command) { - char *bundleID = "com.apple.Terminal"; - AppleEvent evt, res; - AEDesc desc; - OSStatus err; + char *bundleID = "com.apple.Terminal"; + AppleEvent evt, res; + AEDesc desc; + OSStatus err; - [[NSWorkspace sharedWorkspace] launchApplication:@"/Applications/Utilities/Terminal.app/"]; + [[NSWorkspace sharedWorkspace] launchApplication:@"/Applications/Utilities/Terminal.app/"]; - // Build event - err = AEBuildAppleEvent(kAECoreSuite, kAEDoScript, - typeApplicationBundleID, - bundleID, strlen(bundleID), - kAutoGenerateReturnID, - kAnyTransactionID, - &evt, NULL, - "'----':utf8(@)", strlen(command), - command); - if (err) { - NSLog(@"AEBuildAppleEvent failed: %d\n", err); - return err; - } + // Build event + err = AEBuildAppleEvent(kAECoreSuite, kAEDoScript, + typeApplicationBundleID, + bundleID, strlen(bundleID), + kAutoGenerateReturnID, + kAnyTransactionID, + &evt, NULL, + "'----':utf8(@)", strlen(command), + command); + if (err) { + NSLog(@"AEBuildAppleEvent failed: %ld\n", (long)err); + return err; + } - // Send event and check for any Apple Event Manager errors - err = AESendMessage(&evt, &res, kAEWaitReply, kAEDefaultTimeout); - AEDisposeDesc(&evt); - if (err) { - NSLog(@"AESendMessage failed: %d\n", err); - return err; - } - // Check for any application errors - err = AEGetParamDesc(&res, keyErrorNumber, typeSInt32, &desc); - AEDisposeDesc(&res); - if (!err) { - AEGetDescData(&desc, &err, sizeof(err)); - NSLog(@"Terminal returned an error: %d", err); - AEDisposeDesc(&desc); - } else if (err == errAEDescNotFound) { - err = noErr; - } else { - NSLog(@"AEGetPArmDesc returned an error: %d", err); - } + // Send event and check for any Apple Event Manager errors + err = AESendMessage(&evt, &res, kAEWaitReply, kAEDefaultTimeout); + AEDisposeDesc(&evt); + if (err) { + NSLog(@"AESendMessage failed: %ld\n", (long)err); + return err; + } + // Check for any application errors + err = AEGetParamDesc(&res, keyErrorNumber, typeSInt32, &desc); + AEDisposeDesc(&res); + if (!err) { + AEGetDescData(&desc, &err, sizeof(err)); + NSLog(@"Terminal returned an error: %ld", (long)err); + AEDisposeDesc(&desc); + } else if (err == errAEDescNotFound) { + err = noErr; + } else { + NSLog(@"AEGetPArmDesc returned an error: %ld", (long)err); + } - return err; + return err; } diff --git a/Mac/PythonLauncher/main.m b/Mac/PythonLauncher/main.m index 6841433f429..04b4d7321f7 100755 --- a/Mac/PythonLauncher/main.m +++ b/Mac/PythonLauncher/main.m @@ -11,7 +11,7 @@ int main(int argc, const char *argv[]) { - char *home = getenv("HOME"); - if (home) chdir(home); + char *home = getenv("HOME"); + if (home) chdir(home); return NSApplicationMain(argc, argv); } diff --git a/Mac/README b/Mac/README index 555b7ca7030..0a07b5ebc95 100644 --- a/Mac/README +++ b/Mac/README @@ -209,9 +209,11 @@ script to set runtime options. These options can be set once and for all through PythonLauncher's preferences dialog. "BuildApplet.app" creates an applet from a Python script. Drop the script on it -and out comes a full-featured MacOS application. There is much more to this, -to be supplied later. Some useful (but outdated) info can be found in -Mac/Demo. +and out comes a full-featured MacOS application. BuildApplet.app is now +deprecated and has been removed in Python 3. As of OS X 10.8, Xcode 4 no +longer supplies the headers for the deprecated QuickDraw APIs used by +the EasyDialogs module making BuildApplet unusable as an app. It will +not be built by the Mac/Makefile in this case. The commandline scripts /usr/local/bin/python and pythonw can be used to run non-GUI and GUI python scripts from the command line, respectively. diff --git a/Makefile.pre.in b/Makefile.pre.in index 93fb0cf40c8..98711ae62bb 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -437,15 +437,20 @@ $(BUILDPYTHON): Modules/python.o $(LIBRARY) $(LDLIBRARY) Modules/python.o \ $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST) -platform: $(BUILDPYTHON) +platform: $(BUILDPYTHON) pybuilddir.txt $(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print get_platform()+"-"+sys.version[0:3]' >platform +# Create build directory and generate the sysconfig build-time data there. +# pybuilddir.txt contains the name of the build dir and is used for +# sys.path fixup -- see Modules/getpath.c. +pybuilddir.txt: $(BUILDPYTHON) + $(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars # Build the shared modules # Under GNU make, MAKEFLAGS are sorted and normalized; the 's' for # -s, --silent or --quiet is always the first char. # Under BSD make, MAKEFLAGS might be " -s -v x=y". -sharedmods: $(BUILDPYTHON) +sharedmods: $(BUILDPYTHON) pybuilddir.txt @case "$$MAKEFLAGS" in \ *\ -s*|s*) quiet="-q";; \ *) quiet="";; \ @@ -955,7 +960,7 @@ libinstall: build_all $(srcdir)/Lib/$(PLATDIR) $(srcdir)/Modules/xxmodule.c else true; \ fi; \ done - @for i in $(srcdir)/Lib/*.py $(srcdir)/Lib/*.doc $(srcdir)/Lib/*.egg-info ; \ + @for i in $(srcdir)/Lib/*.py `cat pybuilddir.txt`/_sysconfigdata.py $(srcdir)/Lib/*.doc $(srcdir)/Lib/*.egg-info ; \ do \ if test -x $$i; then \ $(INSTALL_SCRIPT) $$i $(DESTDIR)$(LIBDEST); \ @@ -1133,6 +1138,7 @@ sharedinstall: sharedmods --install-scripts=$(BINDIR) \ --install-platlib=$(DESTSHARED) \ --root=$(DESTDIR)/ + -rm $(DESTDIR)$(DESTSHARED)/_sysconfigdata.py* # Here are a couple of targets for MacOSX again, to install a full # framework-based Python. frameworkinstall installs everything, the @@ -1288,11 +1294,12 @@ distclean: clobber Modules/Setup Modules/Setup.local Modules/Setup.config \ Modules/ld_so_aix Modules/python.exp Misc/python.pc -rm -f python*-gdb.py - find $(srcdir) '(' -name '*.fdc' -o -name '*~' \ - -o -name '[@,#]*' -o -name '*.old' \ - -o -name '*.orig' -o -name '*.rej' \ - -o -name '*.bak' ')' \ - -exec rm -f {} ';' + -rm -f pybuilddir.txt + find $(srcdir)/[a-zA-Z]* '(' -name '*.fdc' -o -name '*~' \ + -o -name '[@,#]*' -o -name '*.old' \ + -o -name '*.orig' -o -name '*.rej' \ + -o -name '*.bak' ')' \ + -exec rm -f {} ';' # Check for smelly exported symbols (not starting with Py/_Py) smelly: all diff --git a/Misc/ACKS b/Misc/ACKS index 2d6bd07892f..769624c6cf3 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -84,6 +84,7 @@ Ben Bell Thomas Bellman Alexander “Саша” Belopolsky David Benjamin +Oscar Benjamin Andrew Bennetts Andy Bensky Bennett Benson @@ -96,6 +97,7 @@ Natalia B. Bidart David Binger Dominic Binks Philippe Biondi +Michael Birtwell Stuart Bishop Roy Bixler Jonathan Black @@ -109,6 +111,7 @@ Paul Boddie Matthew Boedicker Robin Boerdijk David Bolen +Wouter Bolsterlee Gawain Bolton Gregory Bond Jurjen Bos @@ -123,6 +126,7 @@ Monty Brandenberg Georg Brandl Christopher Brannon Terrence Brannon +Erik Bray Brian Brazil Dave Brennan Tom Bridgman @@ -169,6 +173,7 @@ Jeffrey Chang Mitch Chapman Greg Chapman Brad Chapman +Yogesh Chaudhari David Chaum Nicolas Chauvat Michael Chermside @@ -228,6 +233,7 @@ Scott David Daniels Ben Darnell Kushal Das Jonathan Dasteel +A. Jesse Jiryu Davis John DeGood Ned Deily Vincent Delft @@ -269,6 +275,7 @@ Josip Dzolonga Maxim Dzumanenko Walter Dörwald Hans Eckardt +David Edelsohn Grant Edwards John Ehresman Eric Eisner @@ -308,6 +315,7 @@ Nils Fischbeck Frederik Fix Matt Fleming Hernán Martínez Foffani +Artem Fokin Arnaud Fontaine Michael Foord Amaury Forgeot d'Arc @@ -315,6 +323,7 @@ Doug Fort John Fouhy Stefan Franke Martin Franklin +Bruce Frederiksen Robin Friedrich Bradley Froehle Ivan Frohne @@ -377,6 +386,7 @@ Paul ten Hagen Rasmus Hahn Peter Haight Václav Haisman +Zbigniew Halas Bob Halley Jesse Hallio Jun Hamano @@ -476,6 +486,7 @@ Geert Jansen Jack Jansen Bill Janssen Thomas Jarosch +Rajagopalasarma Jayakrishnan Drew Jenkins Flemming Kjær Jensen Philip H. Jensen @@ -499,6 +510,7 @@ Jens B. Jorgensen Sijin Joseph Andreas Jung Tattoo Mabonzo K. +Sarah K. Bohuslav Kabrda Bob Kahn Kurt B. Kaiser @@ -540,6 +552,7 @@ Jeff Knupp Greg Kochanski Damon Kohler Marko Kohtala +Vajrasky Kok Guido Kollerie Peter A. Koren Joseph Koshy @@ -592,7 +605,9 @@ Petri Lehtinen Luke Kenneth Casson Leighton Tshepang Lekhonkhobe Marc-André Lemburg +Mateusz Lenik John Lenton +Kostyantyn Leschenko Christopher Tur Lesniewski-Laas Mark Levinson William Lewis @@ -620,10 +635,12 @@ Ray Loyzaga Lukas Lueg Loren Luke Fredrik Lundh +Zhongyue Luo Mark Lutz Jim Lynch Mikael Lyngvig Martin von Löwis +Till Maas Jeff MacDonald John Machin Andrew I MacIntyre @@ -654,6 +671,7 @@ Kirk McDonald Chris McDonough Greg McFarlane Alan McIntyre +Jessica McKellar Michael McLay Mark Mc Mahon Gordon McMillan @@ -672,6 +690,7 @@ Carl Meyer Mike Meyer Piotr Meyer Steven Miale +Jason Michalski Trent Mick Tom Middleton Stan Mihai @@ -733,6 +752,7 @@ Nigel O'Brian Kevin O'Connor Tim O'Malley Zooko O'Whielacronx +Elena Oat Pascal Oberndoerfer Jeffrey Ollie Adam Olsen @@ -761,6 +781,7 @@ Peter Parente Alexandre Parenteau Dan Parisien William Park +Heikki Partanen Harri Pasanen Randy Pausch Samuele Pedroni @@ -788,6 +809,7 @@ Jim St. Pierre Dan Pierson Martijn Pieters François Pinard +Tom Pinckney Zach Pincus Michael Piotrowski Antoine Pitrou @@ -805,6 +827,7 @@ Fernando Pérez Eduardo Pérez Brian Quinlan Anders Qvist +Thomas Rachel Burton Radons Jeff Ramnani Brodie Rao @@ -833,6 +856,7 @@ Nicholas Riley Jean-Claude Rimbault Vlad Riscutia Wes Rishel +Dan Riti Juan M. Bello Rivas Davide Rizzo Anthony Roach @@ -893,6 +917,7 @@ Peter Schneider-Kamp Arvin Schnell Scott Schram Chad J. Schroeder +Christian Schubert Sam Schulenburg Stefan Schwarzer Dietmar Schwertberger @@ -914,6 +939,7 @@ Aman Shah Ha Shao Mark Shannon Richard Shapiro +Vlad Shcherbina Justin Sheehy Charlie Shepherd Bruce Sherwood @@ -924,6 +950,7 @@ John W. Shipman Joel Shprentz Itamar Shtull-Trauring Yue Shuaijie +Terrel Shumway Eric Siegerman Paul Sijben Tim Silk @@ -931,6 +958,8 @@ Michael Simcich Ionel Simionescu Kirill Simonov Nathan Paul Simons +Guilherme Simões +Kyle Simpson Ravi Sinha Janne Sinkkonen Ng Pheng Siong @@ -945,6 +974,7 @@ Roy Smith Rafal Smotrzyk Dirk Soede Paul Sokolovsky +Evgeny Sologubov Cody Somerville Clay Spence Stefan Sperling @@ -995,6 +1025,7 @@ Mikhail Terekhov Victor Terrón Richard M. Tew Tobias Thelen +Févry Thibault Lowe Thiderman Nicolas M. Thiéry James Thomas @@ -1023,6 +1054,7 @@ David Turner Stephen Turner Theodore Turocy Bill Tutt +Fraser Tweedale Doobee R. Tzeck Eren Türkay Lionel Ulmer @@ -1035,6 +1067,7 @@ Case Van Horsen Kyle VanderBeek Atul Varma Dmitry Vasiliev +Sebastian Ortiz Vasquez Alexandre Vassalotti Frank Vercruesse Mike Verdone @@ -1042,11 +1075,13 @@ Jaap Vermeulen Al Vezza Jacques A. Vidrine John Viega +Dino Viehland Kannan Vijayan Kurt Vile Norman Vine Frank Visser Johannes Vogel +Alex Volkov Martijn Vries Niki W. Waibel Wojtek Walczak @@ -1064,6 +1099,7 @@ Aaron Watters Henrik Weber Corran Webster Glyn Webster +Phil Webster Stefan Wehr Zack Weinberg Bob Weiner @@ -1125,7 +1161,9 @@ Moshe Zadka Milan Zamazal Artur Zaprzala Mike Zarnstorff +Yury V. Zaytsev Siebren van der Zee +Nickolai Zeldovich Uwe Zessin Cheng Zhang Tarek Ziadé diff --git a/Misc/NEWS b/Misc/NEWS index 75141c61062..144e80176ed 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -1,7 +1,7 @@ Python News +++++++++++ -What's New in Python 2.7.5? +What's New in Python 2.7.6? =========================== *Release date: XXXX-XX-XX* @@ -9,14 +9,453 @@ What's New in Python 2.7.5? Core and Builtins ----------------- +- Issue #13461: Fix a crash in the "replace" error handler on 64-bit platforms. + Patch by Yogesh Chaudhari. + +- Issue #15866: The xmlcharrefreplace error handler no more produces two XML + entities for a non-BMP character on narrow build. + +- Issue #18184: PyUnicode_FromFormat() and PyUnicode_FromFormatV() now raise + OverflowError when an argument of %c format is out of range. + +- Issue #18137: Detect integer overflow on precision in float.__format__() + and complex.__format__(). + +- Issue #18038: SyntaxError raised during compilation sources with illegal + encoding now always contains an encoding name. + +- Issue #18019: Fix crash in the repr of dictionaries containing their own + views. + +- Issue #18427: str.replace could crash the interpreter with huge strings. + Library ------- -- Issue #17614: IDLE no longer raises exception when quickly closing a file. +- Issue #19158: a rare race in BoundedSemaphore could allow .release() too + often. + +- Issue #18037: 2to3 now escapes '\u' and '\U' in native strings. + +- Issue #19137: The pprint module now correctly formats empty set and frozenset + and instances of set and frozenset subclasses. + +- Issue #16040: CVE-2013-1752: nntplib: Limit maximum line lengths to 2048 to + prevent readline() calls from consuming too much memory. Patch by Jyrki + Pulliainen. + +- Issue #12641: Avoid passing "-mno-cygwin" to the mingw32 compiler, except + when necessary. Patch by Oscar Benjamin. + +- Properly initialize all fields of a SSL object after allocation. + +- Issue #4366: Fix building extensions on all platforms when --enable-shared + is used. + +- Issue #18950: Fix miscellaneous bugs in the sunau module. + Au_read.readframes() now updates current file position and reads correct + number of frames from multichannel stream. Au_write.writeframesraw() now + correctly updates current file position. Au_read and Au_write now correctly + work with file object if start file position is not a zero. + +- Issue #18050: Fixed an incompatibility of the re module with Python 2.7.3 + and older binaries. + +- Issue #19037: The mailbox module now makes all changes to maildir files + before moving them into place, to avoid race conditions with other programs + that may be accessing the maildir directory. + +- Issue #14984: On POSIX systems, when netrc is called without a filename + argument (and therefore is reading the user's $HOME/.netrc file), it now + enforces the same security rules as typical ftp clients: the .netrc file must + be owned by the user that owns the process and must not be readable by any + other user. + +- Issue #17324: Fix http.server's request handling case on trailing '/'. Patch + contributed by Vajrasky Kok. + +- Issue #19018: The heapq.merge() function no longer suppresses IndexError + in the underlying iterables. + +- Issue #18784: The uuid module no more attempts to load libc via ctypes.CDLL, + if all necessary functions are already found in libuuid. + Patch by Evgeny Sologubov. + +- Issue #14971: unittest test discovery no longer gets confused when a function + has a different __name__ than its name in the TestCase class dictionary. + +- Issue #18672: Fixed format specifiers for Py_ssize_t in debugging output in + the _sre module. + +- Issue #18830: inspect.getclasstree() no more produces duplicated entries even + when input list contains duplicates. + +- Issue #18909: Fix _tkinter.tkapp.interpaddr() on Windows 64-bit, don't cast + 64-bit pointer to long (32 bits). + +- Issue #18876: The FileIO.mode attribute now better reflects the actual mode + under which the file was opened. Patch by Erik Bray. + +- Issue #18851: Avoid a double close of subprocess pipes when the child + process fails starting. + +- Issue #18418: After fork(), reinit all threads states, not only active ones. + Patch by A. Jesse Jiryu Davis. + +- Issue #11973: Fix a problem in kevent. The flags and fflags fields are now + properly handled as unsigned. + +- Issue #16809: Fixed some tkinter incompabilities with Tcl/Tk 8.6. + +- Issue #16809: Tkinter's splitlist() and split() methods now accept Tcl_Obj + argument. + +- Issue #17119: Fixed integer overflows when processing large Unicode strings + and tuples in the tkinter module. + +- Issue #15233: Python now guarantees that callables registered with the atexit + module will be called in a deterministic order. + +- Issue #18747: Re-seed OpenSSL's pseudo-random number generator after fork. + A pthread_atfork() parent handler is used to seed the PRNG with pid, time + and some stack data. + +- Issue #8865: Concurrent invocation of select.poll.poll() now raises a + RuntimeError exception. Patch by Christian Schubert. + +- Issue #13461: Fix a crash in the TextIOWrapper.tell method on 64-bit + platforms. Patch by Yogesh Chaudhari. + +- Issue #18777: The ssl module now uses the new CRYPTO_THREADID API of + OpenSSL 1.0.0+ instead of the deprecated CRYPTO id callback function. + +- Issue #18768: Correct doc string of RAND_edg(). Patch by Vajrasky Kok. + +- Issue #18178: Fix ctypes on BSD. dlmalloc.c was compiled twice which broke + malloc weak symbols. + +- Issue #18709: Fix CVE-2013-4238. The SSL module now handles NULL bytes + inside subjectAltName correctly. Formerly the module has used OpenSSL's + GENERAL_NAME_print() function to get the string represention of ASN.1 + strings for ``rfc822Name`` (email), ``dNSName`` (DNS) and + ``uniformResourceIdentifier`` (URI). + +- Issue #18756: Improve error reporting in os.urandom() when the failure + is due to something else than /dev/urandom not existing (for example, + exhausting the file descriptor limit). + +- Fix tkinter regression introduced by the security fix in issue #16248. + +- Issue #18676: Change 'positive' to 'non-negative' in queue.py put and get + docstrings and ValueError messages. Patch by Zhongyue Luo + +- Issue #17998: Fix an internal error in regular expression engine. + +- Issue #17557: Fix os.getgroups() to work with the modified behavior of + getgroups(2) on OS X 10.8. Original patch by Mateusz Lenik. + +- Issue #18455: multiprocessing should not retry connect() with same socket. + +- Issue #18513: Fix behaviour of cmath.rect w.r.t. signed zeros on OS X 10.8 + + gcc. + +- Issue #18101: Tcl.split() now process Unicode strings nested in a tuple as it + do with byte strings. + +- Issue #18347: ElementTree's html serializer now preserves the case of + closing tags. + +- Issue #17261: Ensure multiprocessing's proxies use proper address. + +- Issue #17097: Make multiprocessing ignore EINTR. + +- Issue #18155: The csv module now correctly handles csv files that use + a delimiter character that has a special meaning in regexes, instead of + throwing an exception. + +- Issue #18135: ssl.SSLSocket.write() now raises an OverflowError if the input + string in longer than 2 gigabytes. The ssl module does not support partial + write. + +- Issue #18167: cgi.FieldStorage no longer fails to handle multipart/form-data + when \r\n appears at end of 65535 bytes without other newlines. + +- Issue #17403: urllib.parse.robotparser normalizes the urls before adding to + ruleline. This helps in handling certain types invalid urls in a conservative + manner. Patch contributed by Mher Movsisyan. + +- Implement inequality on weakref.WeakSet. + +- Issue #17981: Closed socket on error in SysLogHandler. + +- Issue #18015: Fix unpickling of 2.7.3 and 2.7.4 namedtuples. + +- Issue #17754: Make ctypes.util.find_library() independent of the locale. + +- Fix typos in the multiprocessing module. + +- Issue #17269: Workaround for socket.getaddrinfo crash on MacOS X + with port None or "0" and flags AI_NUMERICSERV. + +- Issue #18080: When building a C extension module on OS X, if the compiler + is overriden with the CC environment variable, use the new compiler as + the default for linking if LDSHARED is not also overriden. This restores + Distutils behavior introduced in 2.7.3 and inadvertently dropped in 2.7.4. + +- Issue #18071: C extension module builds on OS X could fail with TypeError + if the Xcode command line tools were not installed. + +- Issue #18113: Fixed a refcount leak in the curses.panel module's + set_userptr() method. Reported by Atsuo Ishimoto. + +- Issue #18849: Fixed a Windows-specific tempfile bug where collision with an + existing directory caused mkstemp and related APIs to fail instead of + retrying. Report and fix by Vlad Shcherbina. + + +Tools/Demos +----------- + +- Issue #18873: 2to3 and the findnocoding.py script now detect Python source + code encoding only in comment lines. + +- Issue #18817: Fix a resource warning in Lib/aifc.py demo. + +- Issue #18439: Make patchcheck work on Windows for ACKS, NEWS. + +- Issue #18448: Fix a typo in Demo/newmetaclasses/Eiffel.py. + +- Issue #12990: The "Python Launcher" on OSX could not launch python scripts + that have paths that include wide characters. + +Build +----- + +- Issue #16067: Add description into MSI file to replace installer's temporary name. + +- Issue #18256: Compilation fix for recent AIX releases. Patch by + David Edelsohn. + +- Issue #18098: The deprecated OS X Build Applet.app fails to build on + OS X 10.8 systems because the Apple-deprecated QuickDraw headers have + been removed from Xcode 4. Skip building it in this case. + +IDLE +---- + +- Issue #18873: IDLE now detects Python source code encoding only in comment + lines. + +- Issue #18988: The "Tab" key now works when a word is already autocompleted. + +- Issue #18489: Add tests for SearchEngine. Original patch by Phil Webster. + +- Issue #18429: Format / Format Paragraph, now works when comment blocks + are selected. As with text blocks, this works best when the selection + only includes complete lines. + +- Issue #18226: Add docstrings and unittests for FormatParagraph.py. + Original patches by Todd Rovito and Phil Webster. + +- Issue #18279: Format - Strip trailing whitespace no longer marks a file as + changed when it has not been changed. This fix followed the addition of a + test file originally written by Phil Webster (the issue's main goal). + +- Issue #18539: Calltips now work for float default arguments. + +- Issue #7136: In the Idle File menu, "New Window" is renamed "New File". + Patch by Tal Einat, Roget Serwy, and Todd Rovito. + +- Issue #8515: Set __file__ when run file in IDLE. + Initial patch by Bruce Frederiksen. + +- Issue #5492: Avoid traceback when exiting IDLE caused by a race condition. + +- Issue #17511: Keep IDLE find dialog open after clicking "Find Next". + Original patch by Sarah K. + +- Issue #15392: Create a unittest framework for IDLE. + Preliminary patch by Rajagopalasarma Jayakrishnan + See Lib/idlelib/idle_test/README.txt for how to run Idle tests. + +- Issue #14146: Highlight source line while debugging on Windows. + +- Issue #17532: Always include Options menu for IDLE on OS X. + Patch by Guilherme Simões. + +Tests +----- + +- Issue #18792: Use "127.0.0.1" or "::1" instead of "localhost" as much as + possible, since "localhost" goes through a DNS lookup under recent Windows + versions. + +- Issue #18357: add tests for dictview set difference. + Patch by Fraser Tweedale. + +- Issue #11185: Fix test_wait4 under AIX. Patch by Sébastien Sablé. + +- Issue #18094: test_uuid no more reports skipped tests as passed. + +- Issue #11995: test_pydoc doesn't import all sys.path modules anymore. + +Documentation +------------- + +- Issue #18718: datetime documentation contradictory on leap second support. + +- Issue #17701: Improving strftime documentation. + +- Issue #17844: Refactor a documentation of Python specific encodings. + Add links to encoders and decoders for binary-to-binary codecs. + + +What's New in Python 2.7.5? +=========================== + +*Release date: 2013-05-12* + +Core and Builtins +----------------- + +- Issue #15535: Fixed regression in the pickling of named tuples by + removing the __dict__ property introduced in 2.7.4. + +- Issue #17857: Prevent build failures with pre-3.5.0 versions of sqlite3, + such as was shipped with Centos 5 and Mac OS X 10.4. + +- Issue #17703: Fix a regression where an illegal use of Py_DECREF() after + interpreter finalization can cause a crash. + +- Issue #16447: Fixed potential segmentation fault when setting __name__ on a + class. + +- Issue #17610: Don't rely on non-standard behavior of the C qsort() function. + +Library +------- + +- Issue #17979: Fixed the re module in build with --disable-unicode. + +- Issue #17606: Fixed support of encoded byte strings in the XMLGenerator + .characters() and ignorableWhitespace() methods. Original patch by Sebastian + Ortiz Vasquez. + +- Issue #16601: Restarting iteration over tarfile no more continues from where + it left off. Patch by Michael Birtwell. + +- Issue 16584: in filecomp._cmp, catch IOError as well as os.error. + Patch by Till Maas. + +- Issue #17926: Fix dbm.__contains__ on 64-bit big-endian machines. + +- Issue #17918: When using SSLSocket.accept(), if the SSL handshake failed + on the new socket, the socket would linger indefinitely. Thanks to + Peter Saveliev for reporting. + +- Issue #17289: The readline module now plays nicer with external modules + or applications changing the rl_completer_word_break_characters global + variable. Initial patch by Bradley Froehle. + +- Issue #12181: select module: Fix struct kevent definition on OpenBSD 64-bit + platforms. Patch by Federico Schwindt. + +- Issue #14173: Avoid crashing when reading a signal handler during + interpreter shutdown. + +- Issue #16316: mimetypes now recognizes the .xz and .txz (.tar.xz) extensions. + +- Issue #17192: Restore the patch for Issue #10309 which was ommitted + in 2.7.4 when updating the bundled version of libffi used by ctypes. + +- Issue #17843: Removed test data file that was triggering false-positive virus + warnings with certain antivirus software. + +- Issue #17353: Plistlib emitted empty data tags with deeply nested datastructures + +- Issue #11714: Use 'with' statements to assure a Semaphore releases a + condition variable. Original patch by Thomas Rachel. + +- Issue #17795: Reverted backwards-incompatible change in SysLogHandler with + Unix domain sockets. + +- Issue #17555: Fix ForkAwareThreadLock so that size of after fork + registry does not grow exponentially with generation of process. + +- Issue #17710: Fix cPickle raising a SystemError on bogus input. + +- Issue #17341: Include the invalid name in the error messages from re about + invalid group names. + +- Issue #17016: Get rid of possible pointer wraparounds and integer overflows + in the re module. Patch by Nickolai Zeldovich. + +- Issue #17536: Add to webbrowser's browser list: xdg-open, gvfs-open, + www-browser, x-www-browser, chromium browsers, iceweasel, iceape. + +- Issue #17656: Fix extraction of zip files with unicode member paths. + +- Issue #17666: Fix reading gzip files with an extra field. + +- Issue #13150, #17512: sysconfig no longer parses the Makefile and config.h + files when imported, instead doing it at build time. This makes importing + sysconfig faster and reduces Python startup time by 20%. - Issue #13163: Rename operands in smtplib.SMTP._get_socket to correct names; fixes otherwise misleading output in tracebacks and when when debug is on. +- Issue #17526: fix an IndexError raised while passing code without filename to + inspect.findsource(). Initial patch by Tyler Doyle. + +Build +----- + +- Issue #17547: In configure, explicitly pass -Wformat for the benefit for GCC + 4.8. + +- Issue #17682: Add the _io module to Modules/Setup.dist (commented out). + +- Issue #17086: Search the include and library directories provided by the + compiler. + +Tests +----- + +- Issue #17928: Fix test_structmembers on 64-bit big-endian machines. + +- Issue #17883: Fix buildbot testing of Tkinter on Windows. + Patch by Zachary Ware. + +- Issue #7855: Add tests for ctypes/winreg for issues found in IronPython. + Initial patch by Dino Viehland. + +- Issue #17712: Fix test_gdb failures on Ubuntu 13.04. + +- Issue #17065: Use process-unique key for winreg tests to avoid failures if + test is run multiple times in parallel (eg: on a buildbot host). + +IDLE +---- + +- Issue #17838: Allow sys.stdin to be reassigned. + +- Issue #14735: Update IDLE docs to omit "Control-z on Windows". + +- Issue #17585: Fixed IDLE regression. Now closes when using exit() or quit(). + +- Issue #17657: Show full Tk version in IDLE's about dialog. + Patch by Todd Rovito. + +- Issue #17613: Prevent traceback when removing syntax colorizer in IDLE. + +- Issue #1207589: Backwards-compatibility patch for right-click menu in IDLE. + +- Issue #16887: IDLE now accepts Cancel in tabify/untabify dialog box. + +- Issue #14254: IDLE now handles readline correctly across shell restarts. + +- Issue #17614: IDLE no longer raises exception when quickly closing a file. + - Issue #6698: IDLE now opens just an editor window when configured to do so. - Issue #8900: Using keyboard shortcuts in IDLE to open a file no longer @@ -24,14 +463,19 @@ Library - Issue #6649: Fixed missing exit status in IDLE. Patch by Guilherme Polo. -- Issue #17526: fix an IndexError raised while passing code without filename to - inspect.findsource(). Initial patch by Tyler Doyle. +Documentation +------------- + +- Issue #15940: Specify effect of locale on time functions. + +- Issue #6696: add documentation for the Profile objects, and improve + profile/cProfile docs. Patch by Tom Pinckney. What's New in Python 2.7.4? =========================== -*Release date: XXXX-XX-XX* +*Release date: 2013-04-06* Build ----- @@ -45,8 +489,16 @@ Core and Builtins mapping such that any type with a __getitem__ can be used on the right hand side. -Library -------- +IDLE +---- + +- Issue #17625: In IDLE, close the replace dialog after it is used. + +Tests +----- + +- Issue #17835: Fix test_io when the default OS pipe buffer size is larger + than one million bytes. - Issue #17531: Fix tests that thought group and user ids were always the int type. Also, always allow -1 as a valid group and user id. @@ -183,8 +635,6 @@ Core and Builtins - Issue #15604: Update uses of PyObject_IsTrue() to check for and handle errors correctly. Patch by Serhiy Storchaka. -- Issue #15041: Update "see also" list in tkinter documentation. - - Issue #14579: Fix error handling bug in the utf-16 decoder. Patch by Serhiy Storchaka. @@ -276,7 +726,7 @@ Library - Issue #12718: Fix interaction with winpdb overriding __import__ by setting importer attribute on BaseConfigurator instance. - + - Issue #17521: Corrected non-enabling of logger following two calls to fileConfig(). @@ -349,14 +799,9 @@ Library - Issue #6975: os.path.realpath() now correctly resolves multiple nested symlinks on POSIX platforms. -- Issue #17156: pygettext.py now correctly escapes non-ascii characters. - - Issue #7358: cStringIO.StringIO now supports writing to and reading from a stream larger than 2 GiB on 64-bit systems. -- IDLE was displaying spurious SystemExit tracebacks when running scripts - that terminated by raising SystemExit (i.e. unittest and turtledemo). - - Issue #10355: In SpooledTemporaryFile class mode and name properties and xreadlines method now work for unrolled files. encoding and newlines properties now removed as they have no sense and always produced @@ -408,15 +853,9 @@ Library - Issue #17051: Fix a memory leak in os.path.isdir() on Windows. Patch by Robert Xiao. -- Issue #9290: In IDLE the sys.std* streams now implement io.TextIOBase - interface and support all mandatory methods and properties. - - Issue #13454: Fix a crash when deleting an iterator created by itertools.tee() if all other iterators were very advanced before. -- Issue #1159051: GzipFile now raises EOFError when reading a corrupted file - with truncated header or footer. - - Issue #16992: On Windows in signal.set_wakeup_fd, validate the file descriptor argument. @@ -428,9 +867,6 @@ Library - Issue #9720: zipfile now writes correct local headers for files larger than 4 GiB. -- Issue #16829: IDLE printing no longer fails if there are spaces or other - special characters in the file path. - - Issue #13899: \A, \Z, and \B now correctly match the A, Z, and B literals when used inside character classes (e.g. '[\A]'). Patch by Matthew Barnett. @@ -448,8 +884,6 @@ Library - Issue #16828: Fix error incorrectly raised by bz2.compress(''). Patch by Martin Packman. -- Issue #16819: IDLE method completion now correctly works for unicode literals. - - Issue #9586: Redefine SEM_FAILED on MacOSX to keep compiler happy. - Issue #10527: make multiprocessing use poll() instead of select() if available. @@ -460,12 +894,6 @@ Library - Issue #12065: connect_ex() on an SSL socket now returns the original errno when the socket's timeout expires (it used to return None). -- Issue #16504: IDLE now catches SyntaxErrors raised by tokenizer. Patch by - Roger Serwy. - -- Issue #16702: test_urllib2_localnet tests now correctly ignores proxies for - localhost tests. - - Issue #16713: Fix the parsing of tel url with params using urlparse module. - Issue #16443: Add docstrings to regular expression match objects. @@ -504,8 +932,6 @@ Library list() calls aren't added to filter(), map(), and zip() which are directly passed enumerate(). -- Issue #16476: Fix json.tool to avoid including trailing whitespace. - - Issue #1160: Fix compiling large regular expressions on UCS2 builds. Patch by Serhiy Storchaka. @@ -536,9 +962,6 @@ Library - Issue #16152: fix tokenize to ignore whitespace at the end of the code when no newline is found. Patch by Ned Batchelder. -- Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu - Patch by Todd Rovito. - - Issue #16230: Fix a crash in select.select() when one the lists changes size while iterated on. Patch by Serhiy Storchaka. @@ -624,15 +1047,9 @@ Library - Issue #15424: Add a __sizeof__ implementation for array objects. Patch by Ludwig Hähne. -- Issue #13052: Fix IDLE crashing when replace string in Search/Replace dialog - ended with '\'. Patch by Roger Serwy. - - Issue #15538: Fix compilation of the getnameinfo() / getaddrinfo() emulation code. Patch by Philipp Hagemeister. -- Issue #9803: Don't close IDLE on saving if breakpoint is open. - Patch by Roger Serwy. - - Issue #12288: Consider '0' and '0.0' as valid initialvalue for tkinter SimpleDialog. @@ -707,23 +1124,6 @@ Library - Issue #12157: Make pool.map() empty iterables correctly. Initial patch by mouad. -- Issue #14958: Change IDLE systax highlighting to recognize all string and byte - literals currently supported in Python 2.7. - -- Issue #14962: Update text coloring in IDLE shell window after changing - options. Patch by Roger Serwy. - -- Issue #10997: Prevent a duplicate entry in IDLE's "Recent Files" menu. - -- Issue #12510: Attempting to get invalid tooltip no longer closes Idle. - Original patch by Roger Serwy. - -- Issue #10365: File open dialog now works instead of crashing - even when parent window is closed. Patch by Roger Serwy. - -- Issue #14876: Use user-selected font for highlight configuration. - Patch by Roger Serwy. - - Issue #14036: Add an additional check to validate that port in urlparse does not go in illegal range and returns None. @@ -813,11 +1213,6 @@ Library returned. This avoids crashing the server loop when a signal is received. Patch by Jerzy Kozera. -- Issue #14409: IDLE now properly executes commands in the Shell window - when it cannot read the normal config files on startup and - has to use the built-in default key bindings. - There was previously a bug in one of the defaults. - - Issue #10340: asyncore - properly handle EINVAL in dispatcher constructor on OSX; avoid to call handle_connect in case of a disconnected socket which was not meant to connect. @@ -825,9 +1220,6 @@ Library - Issue #12757: Fix the skipping of doctests when python is run with -OO so that it works in unittest's verbose mode as well as non-verbose mode. -- Issue #3573: IDLE hangs when passing invalid command line args - (directory(ies) instead of file(s)) (Patch by Guilherme Polo) - - Issue #13694: asynchronous connect in asyncore.dispatcher does not set addr attribute. @@ -835,8 +1227,6 @@ Library - Issue #11199: Fix the with urllib which hangs on particular ftp urls. -- Issue #5219: Prevent event handler cascade in IDLE. - - Issue #14252: Fix subprocess.Popen.terminate() to not raise an error under Windows when the child process has already exited. @@ -850,9 +1240,6 @@ Library - Issue #2945: Make the distutils upload command aware of bdist_rpm products. -- Issue #13447: Add a test file to host regression tests for bugs in the - scripts found in the Tools directory. - - Issue #6884: Fix long-standing bugs with MANIFEST.in parsing in distutils on Windows. @@ -904,9 +1291,68 @@ Extension Modules and problematic Apple llvm-gcc compiler. If original compiler is not available, use clang instead by default. +IDLE +---- + +- IDLE was displaying spurious SystemExit tracebacks when running scripts + that terminated by raising SystemExit (i.e. unittest and turtledemo). + +- Issue #9290: In IDLE the sys.std* streams now implement io.TextIOBase + interface and support all mandatory methods and properties. + +- Issue #16829: IDLE printing no longer fails if there are spaces or other + special characters in the file path. + +- Issue #16819: IDLE method completion now correctly works for unicode literals. + +- Issue #16504: IDLE now catches SyntaxErrors raised by tokenizer. Patch by + Roger Serwy. + +- Issue #1207589: Add Cut/Copy/Paste items to IDLE right click Context Menu + Patch by Todd Rovito. + +- Issue #13052: Fix IDLE crashing when replace string in Search/Replace dialog + ended with '\'. Patch by Roger Serwy. + +- Issue #9803: Don't close IDLE on saving if breakpoint is open. + Patch by Roger Serwy. + +- Issue #14958: Change IDLE systax highlighting to recognize all string and byte + literals currently supported in Python 2.7. + +- Issue #14962: Update text coloring in IDLE shell window after changing + options. Patch by Roger Serwy. + +- Issue #10997: Prevent a duplicate entry in IDLE's "Recent Files" menu. + +- Issue #12510: Attempting to get invalid tooltip no longer closes IDLE. + Original patch by Roger Serwy. + +- Issue #10365: File open dialog now works instead of crashing + even when parent window is closed. Patch by Roger Serwy. + +- Issue #14876: Use user-selected font for highlight configuration. + Patch by Roger Serwy. + +- Issue #14409: IDLE now properly executes commands in the Shell window + when it cannot read the normal config files on startup and + has to use the built-in default key bindings. + There was previously a bug in one of the defaults. + +- Issue #3573: IDLE hangs when passing invalid command line args + (directory(ies) instead of file(s)) (Patch by Guilherme Polo) + +- Issue #5219: Prevent event handler cascade in IDLE. + Tests ----- +- Issue #16702: test_urllib2_localnet tests now correctly ignores proxies for + localhost tests. + +- Issue #13447: Add a test file to host regression tests for bugs in the + scripts found in the Tools directory. + - Issue #11420: make test suite pass with -B/DONTWRITEBYTECODE set. Initial patch by Thomas Wouters. @@ -1036,17 +1482,23 @@ Build Tools/Demos ----------- +- Issue #17156: pygettext.py now correctly escapes non-ascii characters. + - Issue #15539: Fix a number of bugs in Tools/scripts/pindent.py. Now pindent.py works with a "with" statement. pindent.py no longer produces improper indentation. pindent.py now works with continued lines broken after "class" or "def" keywords and with continuations at the start of line. +- Issue #16476: Fix json.tool to avoid including trailing whitespace. + - Issue #13301: use ast.literal_eval() instead of eval() in Tools/i18n/msgfmt.py Patch by Serhiy Storchaka. Documentation ------------- +- Issue #15041: Update "see also" list in tkinter documentation. + - Issue #17412: update 2.7 Doc/make.bat to also use sphinx-1.0.7. - Issue #17047: remove doubled words in docs and docstrings @@ -1289,21 +1741,8 @@ Library - Issue #10811: Fix recursive usage of cursors. Instead of crashing, raise a ProgrammingError now. -- Issue #10881: Fix test_site failures with OS X framework builds. - -- Issue #964437 Make IDLE help window non-modal. - Patch by Guilherme Polo and Roger Serwy. - -- Issue #13933: IDLE auto-complete did not work with some imported - module, like hashlib. (Patch by Roger Serwy) - -- Issue #13901: Prevent test_distutils failures on OS X with --enable-shared. - - Issue #13676: Handle strings with embedded zeros correctly in sqlite3. -- Issue #13506: Add '' to path for IDLE Shell when started and restarted with Restart Shell. - Original patches by Marco Scataglini and Roger Serwy. - - Issue #13806: The size check in audioop decompression functions was too strict and could reject valid compressed data. Patch by Oleg Plakhotnyuk. @@ -1342,10 +1781,6 @@ Library - Issue #8035: urllib: Fix a bug where the client could remain stuck after a redirection or an error. -- Issue #4625: If IDLE cannot write to its recent file or breakpoint - files, display a message popup and continue rather than crash. - (original patch by Roger Serwy) - - tarfile.py: Correctly detect bzip2 compressed streams with blocksizes other than 900k. @@ -1375,9 +1810,6 @@ Library node when it is the only child of an element. Initial patch by Dan Kenigsberg. -- Issue #8793: Prevent IDLE crash when given strings with invalid hex escape - sequences. - - Issues #1745761, #755670, #13357, #12629, #1200313: HTMLParser now correctly handles non-valid attributes, including adjacent and unquoted attributes. @@ -1400,9 +1832,6 @@ Library - Issue #10817: Fix urlretrieve function to raise ContentTooShortError even when reporthook is None. Patch by Jyrki Pulliainen. -- Issue #13296: Fix IDLE to clear compile __future__ flags on shell restart. - (Patch by Roger Serwy) - - Issue #7334: close source files on ElementTree.parse and iterparse. - Issue #13232: logging: Improved logging of exceptions in the presence of @@ -1647,6 +2076,28 @@ Extension Modules signature. Without this, architectures where sizeof void* != sizeof int are broken. Patch given by Hallvard B Furuseth. +IDLE +---- + +- Issue #964437 Make IDLE help window non-modal. + Patch by Guilherme Polo and Roger Serwy. + +- Issue #13933: IDLE auto-complete did not work with some imported + module, like hashlib. (Patch by Roger Serwy) + +- Issue #13506: Add '' to path for IDLE Shell when started and restarted with Restart Shell. + Original patches by Marco Scataglini and Roger Serwy. + +- Issue #4625: If IDLE cannot write to its recent file or breakpoint + files, display a message popup and continue rather than crash. + (original patch by Roger Serwy) + +- Issue #8793: Prevent IDLE crash when given strings with invalid hex escape + sequences. + +- Issue #13296: Fix IDLE to clear compile __future__ flags on shell restart. + (Patch by Roger Serwy) + Build ----- @@ -1687,6 +2138,10 @@ Tests - Issue #11689: Fix a variable scoping error in an sqlite3 test. Initial patch by Torsten Landschoff. +- Issue #10881: Fix test_site failures with OS X framework builds. + +- Issue #13901: Prevent test_distutils failures on OS X with --enable-shared. + - Issue #13304: Skip test case if user site-packages disabled (-s or PYTHONNOUSERSITE). (Patch by Carl Meyer) @@ -1859,9 +2314,6 @@ Core and Builtins Library ------- -- Issue #12590: IDLE editor window now always displays the first line - when opening a long file. With Tk 8.5, the first line was hidden. - - Issue #12161: Cause StringIO.getvalue() to raise a ValueError when used on a closed StringIO instance. @@ -1883,9 +2335,6 @@ Library - Issue #12124: zipimport doesn't keep a reference to zlib.decompress() anymore to be able to unload the module. -- Issue #11088: don't crash when using F5 to run a script in IDLE on MacOSX - with Tk 8.5. - - Issue #10154, #10090: change the normalization of UTF-8 to "UTF-8" instead of "UTF8" in the locale module as the latter is not supported MacOSX and OpenBSD. @@ -1905,8 +2354,6 @@ Library - Issue #12012: ssl.PROTOCOL_SSLv2 becomes optional. -- Issue #11164: Remove obsolete allnodes test from minidom test. - - Issue #11927: SMTP_SSL now uses port 465 by default as documented. Patch by Kasun Herath. @@ -2058,17 +2505,6 @@ Library - Issue #8275: Fix passing of callback arguments with ctypes under Win64. Patch by Stan Mihai. -- Issue #10940: Workaround an IDLE hang on Mac OS X 10.6 when using the - menu accelerators for Open Module, Go to Line, and New Indent Width. - The accelerators still work but no longer appear in the menu items. - -- Issue #10907: Warn OS X 10.6 IDLE users to use ActiveState Tcl/Tk 8.5, rather - than the currently problematic Apple-supplied one, when running with the - 64-/32-bit installer variant. - -- Issue #11052: Correct IDLE menu accelerators on Mac OS X for Save - commands. - - Issue #10949: Improved robustness of rotating file handlers. - Issue #10955: Fix a potential crash when trying to mmap() a file past its @@ -2077,9 +2513,6 @@ Library - Issue #10898: Allow compiling the posix module when the C library defines a symbol named FSTAT. -- Issue #6075: IDLE on Mac OS X now works with both Carbon AquaTk and - Cocoa AquaTk. - - Issue #10916: mmap should not segfault when a file is mapped using 0 as length and a non-zero offset, and an attempt to read past the end of file is made (IndexError is raised instead). Patch by Ross Lagerwall. @@ -2138,8 +2571,6 @@ Library - Issue #6791: Limit header line length (to 65535 bytes) in http.client, to avoid denial of services from the other party. -- Issue #10404: Use ctl-button-1 on OSX for the context menu in Idle. - - Issue #9907: Fix tab handling on OSX when using editline by calling rl_initialize first, then setting our custom defaults, then reading .editrc. @@ -2157,11 +2588,6 @@ Library - Issue #10695: passing the port as a string value to telnetlib no longer causes debug mode to fail. -- Issue #10107: Warn about unsaved files in IDLE on OSX. - -- Issue #10406: Enable Rstrip IDLE extension on OSX (just like on other - platforms). - - Issue #10478: Reentrant calls inside buffered IO objects (for example by way of a signal handler) now raise a RuntimeError instead of freezing the current process. @@ -2208,6 +2634,39 @@ Extension Modules - Issue #678250: Make mmap flush a noop on ACCESS_READ and ACCESS_COPY. +IDLE +---- + +- Issue #11718: IDLE's open module dialog couldn't find the __init__.py + file in a package. + +- Issue #12590: IDLE editor window now always displays the first line + when opening a long file. With Tk 8.5, the first line was hidden. + +- Issue #11088: don't crash when using F5 to run a script in IDLE on MacOSX + with Tk 8.5. + +- Issue #10940: Workaround an IDLE hang on Mac OS X 10.6 when using the + menu accelerators for Open Module, Go to Line, and New Indent Width. + The accelerators still work but no longer appear in the menu items. + +- Issue #10907: Warn OS X 10.6 IDLE users to use ActiveState Tcl/Tk 8.5, rather + than the currently problematic Apple-supplied one, when running with the + 64-/32-bit installer variant. + +- Issue #11052: Correct IDLE menu accelerators on Mac OS X for Save + commands. + +- Issue #6075: IDLE on Mac OS X now works with both Carbon AquaTk and + Cocoa AquaTk. + +- Issue #10404: Use ctl-button-1 on OSX for the context menu in Idle. + +- Issue #10107: Warn about unsaved files in IDLE on OSX. + +- Issue #10406: Enable Rstrip IDLE extension on OSX (just like on other + platforms). + Build ----- @@ -2253,15 +2712,11 @@ Build - Issue #1099: Fix the build on MacOSX when building a framework with pydebug using GCC 4.0. -IDLE ----- - -- Issue #11718: IDLE's open module dialog couldn't find the __init__.py - file in a package. - Tests ----- +- Issue #11164: Remove obsolete allnodes test from minidom test. + - Issue #12205: Fix test_subprocess failure due to uninstalled test data. - Issue #5723: Improve json tests to be executed with and without accelerations. @@ -2330,9 +2785,6 @@ Library - Issue #4493: urllib2 adds '/' in front of path components which does not start with '/. Common behavior exhibited by browsers and other clients. -- Issue #6378: idle.bat now runs with the appropriate Python version rather than - the system default. Patch by Sridhar Ratnakumar. - - Issue #10407: Fix one NameError in distutils. - Issue #10198: fix duplicate header written to wave files when writeframes() @@ -2343,6 +2795,12 @@ Library - Issue #5111: IPv6 Host in the Header is wrapped inside [ ]. Patch by Chandru. +IDLE +---- + +- Issue #6378: idle.bat now runs with the appropriate Python version rather than + the system default. Patch by Sridhar Ratnakumar. + Build ----- @@ -3108,7 +3566,7 @@ Library - Issue #1285086: Speed up ``urllib.quote()`` and urllib.unquote for simple cases. -- Issue #8688: Distutils now recalculates MANIFEST everytime. +- Issue #8688: Distutils now recalculates MANIFEST every time. - Issue #5099: The ``__del__()`` method of ``subprocess.Popen`` (and the methods it calls) referenced global objects, causing errors to pop up during @@ -4992,9 +5450,6 @@ Library - Issue #6048: Now Distutils uses the tarfile module in archive_util. -- Issue #5150: IDLE's format menu now has an option to strip trailing - whitespace. - - Issue #6121: pydoc now ignores leading and trailing spaces in the argument to the 'help' function. @@ -5653,6 +6108,14 @@ Library - Windows locale mapping updated to Vista. +IDLE +---- + +- Issue #5150: IDLE's format menu now has an option to strip trailing + whitespace. + +- Issue #5847: Remove -n switch on "Edit with IDLE" menu item. + Tools/Demos ----------- @@ -5686,8 +6149,6 @@ Build - Issue #6094: Build correctly with Subversion 1.7. -- Issue #5847: Remove -n switch on "Edit with IDLE" menu item. - - Issue #5726: Make Modules/ld_so_aix return the actual exit code of the linker, rather than always exit successfully. Patch by Floris Bruynooghe. @@ -5961,7 +6422,7 @@ Library - Issue #3547: Fixed ctypes structures bitfields of varying integer sizes. -- Issue #3879: A regression in urllib.getproxies_enviroment was fixed. +- Issue #3879: A regression in urllib.getproxies_environment was fixed. - Issue #3863: Disabled a unit test of fork being called from a thread when running on platforms known to exhibit OS bugs when attempting that. @@ -8507,9 +8968,6 @@ Library Allows the demo2 function to be executed on its own instead of only when the module is run as a script. -- Bug #813342: Start the IDLE subprocess with -Qnew if the parent is - started with that option. - - Bug #1565150: Fix subsecond processing for os.utime on Windows. - Support for MSVC 8 was added to bdist_wininst. @@ -8558,9 +9016,6 @@ Library - Bug #1531862: Do not close standard file descriptors in subprocess. -- idle: Honor the "Cancel" action in the save dialog (Debian bug - #299092). - - Fix utf-8-sig incremental decoder, which didn't recognise a BOM when the first chunk fed to the decoder started with a BOM, but was longer than 3 bytes. @@ -8803,6 +9258,15 @@ Extension Modules - The sqlite3 module was updated to pysqlite 2.4.1. +IDLE +---- + +- Bug #813342: Start the IDLE subprocess with -Qnew if the parent is + started with that option. + +- IDLE: Honor the "Cancel" action in the save dialog (Debian bug + #299092). + Tests ----- diff --git a/Misc/RPM/python-2.7.spec b/Misc/RPM/python-2.7.spec index 0b62adfe738..9df3b2c88e9 100644 --- a/Misc/RPM/python-2.7.spec +++ b/Misc/RPM/python-2.7.spec @@ -39,7 +39,7 @@ %define name python #--start constants-- -%define version 2.7.4rc1 +%define version 2.7.5 %define libvers 2.7 #--end constants-- %define release 1pydotorg diff --git a/Modules/Setup.dist b/Modules/Setup.dist index 1121ae64a84..2ad1aa33d33 100644 --- a/Modules/Setup.dist +++ b/Modules/Setup.dist @@ -191,6 +191,9 @@ GLHACK=-Dclear=__GLclear # access to ISO C locale support #_locale _localemodule.c # -lintl +# Standard I/O baseline +#_io -I$(srcdir)/Modules/_io _io/bufferedio.c _io/bytesio.c _io/fileio.c _io/iobase.c _io/_iomodule.c _io/stringio.c _io/textio.c + # Modules with some UNIX dependencies -- on by default: # (If you have a really backward UNIX, select and socket may not be @@ -208,7 +211,7 @@ GLHACK=-Dclear=__GLclear #_csv _csv.c # Socket module helper for socket(2) -#_socket socketmodule.c +#_socket socketmodule.c timemodule.c # Socket module helper for SSL support; you must comment out the other # socket line above, and possibly edit the SSL variable: diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 371631ce266..26d878344e1 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -8,9 +8,13 @@ */ /* The block length may be set to any number over 1. Larger numbers - * reduce the number of calls to the memory allocator but take more - * memory. Ideally, BLOCKLEN should be set with an eye to the - * length of a cache line. + * reduce the number of calls to the memory allocator, give faster + * indexing and rotation, and reduce the link::data overhead ratio. + * + * Ideally, the block length will be set to two less than some + * multiple of the cache-line length (so that the full block + * including the leftlink and rightlink will fit neatly into + * cache lines). */ #define BLOCKLEN 62 @@ -46,9 +50,9 @@ */ typedef struct BLOCK { - struct BLOCK *leftlink; - struct BLOCK *rightlink; PyObject *data[BLOCKLEN]; + struct BLOCK *rightlink; + struct BLOCK *leftlink; } block; #define MAXFREEBLOCKS 10 @@ -58,13 +62,8 @@ static block *freeblocks[MAXFREEBLOCKS]; static block * newblock(block *leftlink, block *rightlink, Py_ssize_t len) { block *b; - /* To prevent len from overflowing PY_SSIZE_T_MAX on 64-bit machines, we - * refuse to allocate new blocks if the current len is dangerously - * close. There is some extra margin to prevent spurious arithmetic - * overflows at various places. The following check ensures that - * the blocks allocated to the deque, in the worst case, can only - * have PY_SSIZE_T_MAX-2 entries in total. - */ + /* To prevent len from overflowing PY_SSIZE_T_MAX on 32-bit machines, we + * refuse to allocate new blocks if the current len is nearing overflow. */ if (len >= PY_SSIZE_T_MAX - 2*BLOCKLEN) { PyErr_SetString(PyExc_OverflowError, "cannot add more blocks to the deque"); @@ -103,8 +102,8 @@ typedef struct { Py_ssize_t leftindex; /* in range(BLOCKLEN) */ Py_ssize_t rightindex; /* in range(BLOCKLEN) */ Py_ssize_t len; - Py_ssize_t maxlen; long state; /* incremented whenever the indices move */ + Py_ssize_t maxlen; PyObject *weakreflist; /* List of weak references */ } dequeobject; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 6daf455a06e..6642dc3f4c0 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -75,6 +75,10 @@ #include #include "ctypes.h" +#ifdef HAVE_ALLOCA_H +/* AIX needs alloca.h for alloca() */ +#include +#endif #if defined(_DEBUG) || defined(__MINGW32__) /* Don't use structured exception handling on Windows if this is defined. diff --git a/Modules/_ctypes/libffi/fficonfig.py.in b/Modules/_ctypes/libffi/fficonfig.py.in index 27c971f86f2..c9108148761 100644 --- a/Modules/_ctypes/libffi/fficonfig.py.in +++ b/Modules/_ctypes/libffi/fficonfig.py.in @@ -1,7 +1,6 @@ ffi_sources = """ src/prep_cif.c src/closures.c -src/dlmalloc.c """.split() ffi_platforms = { diff --git a/Modules/_ctypes/libffi/src/dlmalloc.c b/Modules/_ctypes/libffi/src/dlmalloc.c index 5c9f9c2d239..2773953590e 100644 --- a/Modules/_ctypes/libffi/src/dlmalloc.c +++ b/Modules/_ctypes/libffi/src/dlmalloc.c @@ -457,6 +457,11 @@ DEFAULT_MMAP_THRESHOLD default: 256K #define LACKS_ERRNO_H #define MALLOC_FAILURE_ACTION #define MMAP_CLEARS 0 /* WINCE and some others apparently don't clear */ +#elif !defined _GNU_SOURCE +/* mremap() on Linux requires this via sys/mman.h + * See roundup issue 10309 + */ +#define _GNU_SOURCE 1 #endif /* WIN32 */ #ifdef __OS2__ diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index 04a0a28a885..bdd5cf0177e 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -293,9 +293,18 @@ PyCursesPanel_replace_panel(PyCursesPanelObject *self, PyObject *args) static PyObject * PyCursesPanel_set_panel_userptr(PyCursesPanelObject *self, PyObject *obj) { + PyObject *oldobj; + int rc; + PyCursesInitialised; Py_INCREF(obj); - return PyCursesCheckERR(set_panel_userptr(self->pan, (void*)obj), - "set_panel_userptr"); + oldobj = (PyObject *) panel_userptr(self->pan); + rc = set_panel_userptr(self->pan, (void*)obj); + if (rc == ERR) { + /* In case of an ncurses error, decref the new object again */ + Py_DECREF(obj); + } + Py_XDECREF(oldobj); + return PyCursesCheckERR(rc, "set_panel_userptr"); } static PyObject * diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 379aa01d0a5..b9abcac8d0c 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -2427,6 +2427,8 @@ expat_unknown_encoding_handler(XMLParserObject *self, const XML_Char *name, if (PyUnicode_GET_SIZE(u) != 256) { Py_DECREF(u); + PyErr_SetString(PyExc_ValueError, + "multi-byte encodings are not supported"); return XML_STATUS_ERROR; } diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 6cd7d817c8d..5946e6a2bcc 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -47,6 +47,7 @@ typedef struct { int fd; unsigned int readable : 1; unsigned int writable : 1; + unsigned int appending : 1; signed int seekable : 2; /* -1 means unknown */ unsigned int closefd : 1; PyObject *weakreflist; @@ -124,6 +125,7 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->fd = -1; self->readable = 0; self->writable = 0; + self->appending = 0; self->seekable = -1; self->closefd = 1; self->weakreflist = NULL; @@ -184,7 +186,7 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) Py_UNICODE *widename = NULL; #endif int ret = 0; - int rwa = 0, plus = 0, append = 0; + int rwa = 0, plus = 0; int flags = 0; int fd = -1; int closefd = 1; @@ -279,8 +281,8 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) goto bad_mode; rwa = 1; self->writable = 1; - flags |= O_CREAT; - append = 1; + self->appending = 1; + flags |= O_APPEND | O_CREAT; break; case 'b': break; @@ -311,11 +313,6 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) flags |= O_BINARY; #endif -#ifdef O_APPEND - if (append) - flags |= O_APPEND; -#endif - if (fd >= 0) { if (check_fd(fd)) goto error; @@ -356,7 +353,7 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) if (PyObject_SetAttrString((PyObject *)self, "name", nameobj) < 0) goto error; - if (append) { + if (self->appending) { /* For consistent behaviour, we explicitly seek to the end of file (otherwise, it might be done only on the first write()). */ @@ -898,7 +895,13 @@ fileio_truncate(fileio *self, PyObject *args) static char * mode_string(fileio *self) { - if (self->readable) { + if (self->appending) { + if (self->readable) + return "ab+"; + else + return "ab"; + } + else if (self->readable) { if (self->writable) return "rb+"; else diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index cd6d443e23f..680275876e5 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -2271,7 +2271,7 @@ textiowrapper_tell(textio *self, PyObject *args) int dec_flags; PyObject *decoded = PyObject_CallMethod( - self->decoder, "decode", "s#", input, 1); + self->decoder, "decode", "s#", input, (Py_ssize_t)1); if (check_decoded(decoded) < 0) goto fail; chars_decoded += PyUnicode_GET_SIZE(decoded); diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index c831637f901..d192a074ba5 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -63,7 +63,7 @@ mp_SetError(PyObject *Type, int num) break; default: PyErr_Format(PyExc_RuntimeError, - "unkown error number %d", num); + "unknown error number %d", num); } return NULL; } diff --git a/Modules/_multiprocessing/socket_connection.c b/Modules/_multiprocessing/socket_connection.c index 66bc3777924..bdb0a32e375 100644 --- a/Modules/_multiprocessing/socket_connection.c +++ b/Modules/_multiprocessing/socket_connection.c @@ -22,6 +22,21 @@ # define CLOSE(h) close(h) #endif +/* + * Wrapper for PyErr_CheckSignals() which can be called without the GIL + */ + +static int +check_signals(void) +{ + PyGILState_STATE state; + int res; + state = PyGILState_Ensure(); + res = PyErr_CheckSignals(); + PyGILState_Release(state); + return res; +} + /* * Send string to file descriptor */ @@ -34,8 +49,14 @@ _conn_sendall(HANDLE h, char *string, size_t length) while (length > 0) { res = WRITE(h, p, length); - if (res < 0) + if (res < 0) { + if (errno == EINTR) { + if (check_signals() < 0) + return MP_EXCEPTION_HAS_BEEN_SET; + continue; + } return MP_SOCKET_ERROR; + } length -= res; p += res; } @@ -56,12 +77,16 @@ _conn_recvall(HANDLE h, char *buffer, size_t length) while (remaining > 0) { temp = READ(h, p, remaining); - if (temp <= 0) { - if (temp == 0) - return remaining == length ? - MP_END_OF_FILE : MP_EARLY_END_OF_FILE; - else - return temp; + if (temp < 0) { + if (errno == EINTR) { + if (check_signals() < 0) + return MP_EXCEPTION_HAS_BEEN_SET; + continue; + } + return temp; + } + else if (temp == 0) { + return remaining == length ? MP_END_OF_FILE : MP_EARLY_END_OF_FILE; } remaining -= temp; p += temp; @@ -171,9 +196,16 @@ conn_poll(ConnectionObject *conn, double timeout, PyThreadState *_save) p.revents = 0; if (timeout < 0) { - res = poll(&p, 1, -1); + do { + res = poll(&p, 1, -1); + } while (res < 0 && errno == EINTR); } else { res = poll(&p, 1, (int)(timeout * 1000 + 0.5)); + if (res < 0 && errno == EINTR) { + /* We were interrupted by a signal. Just indicate a + timeout even though we are early. */ + return FALSE; + } } if (res < 0) { @@ -209,12 +241,19 @@ conn_poll(ConnectionObject *conn, double timeout, PyThreadState *_save) FD_SET((SOCKET)conn->handle, &rfds); if (timeout < 0.0) { - res = select((int)conn->handle+1, &rfds, NULL, NULL, NULL); + do { + res = select((int)conn->handle+1, &rfds, NULL, NULL, NULL); + } while (res < 0 && errno == EINTR); } else { struct timeval tv; tv.tv_sec = (long)timeout; tv.tv_usec = (long)((timeout - tv.tv_sec) * 1e6 + 0.5); res = select((int)conn->handle+1, &rfds, NULL, NULL, &tv); + if (res < 0 && errno == EINTR) { + /* We were interrupted by a signal. Just indicate a + timeout even though we are early. */ + return FALSE; + } } if (res < 0) { diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index ae810dea30b..f06f92c9c3b 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -732,7 +732,7 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* Py_DECREF(self->lastrowid); if (!multiple && statement_type == STATEMENT_INSERT) { - sqlite3_int64 lastrowid; + sqlite_int64 lastrowid; Py_BEGIN_ALLOW_THREADS lastrowid = sqlite3_last_insert_rowid(self->connection->db); Py_END_ALLOW_THREADS diff --git a/Modules/_sqlite/util.c b/Modules/_sqlite/util.c index 5f06b66638b..a24dd8c6347 100644 --- a/Modules/_sqlite/util.c +++ b/Modules/_sqlite/util.c @@ -111,7 +111,7 @@ int _pysqlite_seterror(sqlite3* db, sqlite3_stmt* st) #endif PyObject * -_pysqlite_long_from_int64(sqlite3_int64 value) +_pysqlite_long_from_int64(sqlite_int64 value) { #ifdef HAVE_LONG_LONG # if SIZEOF_LONG_LONG < 8 @@ -135,7 +135,7 @@ _pysqlite_long_from_int64(sqlite3_int64 value) return PyInt_FromLong(value); } -sqlite3_int64 +sqlite_int64 _pysqlite_long_as_int64(PyObject * py_val) { int overflow; @@ -158,8 +158,8 @@ _pysqlite_long_as_int64(PyObject * py_val) #endif return value; } - else if (sizeof(value) < sizeof(sqlite3_int64)) { - sqlite3_int64 int64val; + else if (sizeof(value) < sizeof(sqlite_int64)) { + sqlite_int64 int64val; if (_PyLong_AsByteArray((PyLongObject *)py_val, (unsigned char *)&int64val, sizeof(int64val), IS_LITTLE_ENDIAN, 1 /* signed */) >= 0) { diff --git a/Modules/_sqlite/util.h b/Modules/_sqlite/util.h index 12a5710ff31..4c3b2c70cb2 100644 --- a/Modules/_sqlite/util.h +++ b/Modules/_sqlite/util.h @@ -36,7 +36,7 @@ int pysqlite_step(sqlite3_stmt* statement, pysqlite_Connection* connection); */ int _pysqlite_seterror(sqlite3* db, sqlite3_stmt* st); -PyObject * _pysqlite_long_from_int64(sqlite3_int64 value); -sqlite3_int64 _pysqlite_long_as_int64(PyObject * value); +PyObject * _pysqlite_long_from_int64(sqlite_int64 value); +sqlite_int64 _pysqlite_long_as_int64(PyObject * value); #endif diff --git a/Modules/_sre.c b/Modules/_sre.c index f3a29931a7a..a7103cc456b 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -272,7 +272,7 @@ data_stack_grow(SRE_STATE* state, Py_ssize_t size) if (cursize < minsize) { void* stack; cursize = minsize+minsize/4+1024; - TRACE(("allocate/grow stack %d\n", cursize)); + TRACE(("allocate/grow stack %" PY_FORMAT_SIZE_T "d\n", cursize)); stack = PyMem_REALLOC(state->data_stack, cursize); if (!stack) { data_stack_dealloc(state); @@ -592,12 +592,13 @@ SRE_COUNT(SRE_STATE* state, SRE_CODE* pattern, Py_ssize_t maxcount) if (!i) break; } - TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, + TRACE(("|%p|%p|COUNT %" PY_FORMAT_SIZE_T "d\n", pattern, ptr, (SRE_CHAR*) state->ptr - ptr)); return (SRE_CHAR*) state->ptr - ptr; } - TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, ptr - (SRE_CHAR*) state->ptr)); + TRACE(("|%p|%p|COUNT %" PY_FORMAT_SIZE_T "d\n", pattern, ptr, + ptr - (SRE_CHAR*) state->ptr)); return ptr - (SRE_CHAR*) state->ptr; } @@ -684,9 +685,10 @@ SRE_INFO(SRE_STATE* state, SRE_CODE* pattern) #define DATA_STACK_ALLOC(state, type, ptr) \ do { \ alloc_pos = state->data_stack_base; \ - TRACE(("allocating %s in %d (%d)\n", \ + TRACE(("allocating %s in %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ SFY(type), alloc_pos, sizeof(type))); \ - if (state->data_stack_size < alloc_pos+sizeof(type)) { \ + if (sizeof(type) > state->data_stack_size - alloc_pos) { \ int j = data_stack_grow(state, sizeof(type)); \ if (j < 0) return j; \ if (ctx_pos != -1) \ @@ -698,15 +700,16 @@ do { \ #define DATA_STACK_LOOKUP_AT(state, type, ptr, pos) \ do { \ - TRACE(("looking up %s at %d\n", SFY(type), pos)); \ + TRACE(("looking up %s at %" PY_FORMAT_SIZE_T "d\n", SFY(type), pos)); \ ptr = (type*)(state->data_stack+pos); \ } while (0) #define DATA_STACK_PUSH(state, data, size) \ do { \ - TRACE(("copy data in %p to %d (%d)\n", \ + TRACE(("copy data in %p to %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ data, state->data_stack_base, size)); \ - if (state->data_stack_size < state->data_stack_base+size) { \ + if (size > state->data_stack_size - state->data_stack_base) { \ int j = data_stack_grow(state, size); \ if (j < 0) return j; \ if (ctx_pos != -1) \ @@ -718,7 +721,8 @@ do { \ #define DATA_STACK_POP(state, data, size, discard) \ do { \ - TRACE(("copy data to %p from %d (%d)\n", \ + TRACE(("copy data to %p from %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ data, state->data_stack_base-size, size)); \ memcpy(data, state->data_stack+state->data_stack_base-size, size); \ if (discard) \ @@ -727,7 +731,8 @@ do { \ #define DATA_STACK_POP_DISCARD(state, size) \ do { \ - TRACE(("discard data from %d (%d)\n", \ + TRACE(("discard data from %" PY_FORMAT_SIZE_T "d " \ + "(%" PY_FORMAT_SIZE_T "d)\n", \ state->data_stack_base-size, size)); \ state->data_stack_base -= size; \ } while(0) @@ -831,8 +836,9 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) /* optimization info block */ /* <1=skip> <2=flags> <3=min> ... */ if (ctx->pattern[3] && (end - ctx->ptr) < ctx->pattern[3]) { - TRACE(("reject (got %d chars, need %d)\n", - (end - ctx->ptr), ctx->pattern[3])); + TRACE(("reject (got %" PY_FORMAT_SIZE_T "d chars, " + "need %" PY_FORMAT_SIZE_T "d)\n", + (end - ctx->ptr), (Py_ssize_t) ctx->pattern[3])); RETURN_FAILURE; } ctx->pattern += ctx->pattern[1] + 1; @@ -1028,7 +1034,7 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) TRACE(("|%p|%p|REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, ctx->pattern[1], ctx->pattern[2])); - if (ctx->ptr + ctx->pattern[1] > end) + if ((Py_ssize_t) ctx->pattern[1] > end - ctx->ptr) RETURN_FAILURE; /* cannot match */ state->ptr = ctx->ptr; @@ -1111,7 +1117,7 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) TRACE(("|%p|%p|MIN_REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, ctx->pattern[1], ctx->pattern[2])); - if (ctx->ptr + ctx->pattern[1] > end) + if ((Py_ssize_t) ctx->pattern[1] > end - ctx->ptr) RETURN_FAILURE; /* cannot match */ state->ptr = ctx->ptr; @@ -1207,10 +1213,10 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) ctx->count = ctx->u.rep->count+1; - TRACE(("|%p|%p|MAX_UNTIL %d\n", ctx->pattern, + TRACE(("|%p|%p|MAX_UNTIL %" PY_FORMAT_SIZE_T "d\n", ctx->pattern, ctx->ptr, ctx->count)); - if (ctx->count < ctx->u.rep->pattern[1]) { + if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { /* not enough matches */ ctx->u.rep->count = ctx->count; DO_JUMP(JUMP_MAX_UNTIL_1, jump_max_until_1, @@ -1224,7 +1230,7 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) RETURN_FAILURE; } - if ((ctx->count < ctx->u.rep->pattern[2] || + if ((ctx->count < (Py_ssize_t) ctx->u.rep->pattern[2] || ctx->u.rep->pattern[2] == SRE_MAXREPEAT) && state->ptr != ctx->u.rep->last_ptr) { /* we may have enough matches, but if we can @@ -1270,10 +1276,10 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) ctx->count = ctx->u.rep->count+1; - TRACE(("|%p|%p|MIN_UNTIL %d %p\n", ctx->pattern, + TRACE(("|%p|%p|MIN_UNTIL %" PY_FORMAT_SIZE_T "d %p\n", ctx->pattern, ctx->ptr, ctx->count, ctx->u.rep->pattern)); - if (ctx->count < ctx->u.rep->pattern[1]) { + if (ctx->count < (Py_ssize_t) ctx->u.rep->pattern[1]) { /* not enough matches */ ctx->u.rep->count = ctx->count; DO_JUMP(JUMP_MIN_UNTIL_1, jump_min_until_1, @@ -1302,7 +1308,7 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) LASTMARK_RESTORE(); - if ((ctx->count >= ctx->u.rep->pattern[2] + if ((ctx->count >= (Py_ssize_t) ctx->u.rep->pattern[2] && ctx->u.rep->pattern[2] != SRE_MAXREPEAT) || state->ptr == ctx->u.rep->last_ptr) RETURN_FAILURE; @@ -1483,7 +1489,8 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) TRACE(("|%p|%p|JUMP_ASSERT_NOT\n", ctx->pattern, ctx->ptr)); goto jump_assert_not; case JUMP_NONE: - TRACE(("|%p|%p|RETURN %d\n", ctx->pattern, ctx->ptr, ret)); + TRACE(("|%p|%p|RETURN %" PY_FORMAT_SIZE_T "d\n", ctx->pattern, + ctx->ptr, ret)); break; } @@ -1532,7 +1539,8 @@ SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern) pattern += 1 + pattern[1]; } - TRACE(("prefix = %p %d %d\n", prefix, prefix_len, prefix_skip)); + TRACE(("prefix = %p %" PY_FORMAT_SIZE_T "d %" PY_FORMAT_SIZE_T "d\n", + prefix, prefix_len, prefix_skip)); TRACE(("charset = %p\n", charset)); #if defined(USE_FAST_SEARCH) @@ -2784,7 +2792,7 @@ _compile(PyObject* self_, PyObject* args) skip = *code; \ VTRACE(("%lu (skip to %p)\n", \ (unsigned long)skip, code+skip)); \ - if (code+skip-adj < code || code+skip-adj > end)\ + if (skip-adj > end-code) \ FAIL; \ code++; \ } while (0) @@ -2817,7 +2825,7 @@ _validate_charset(SRE_CODE *code, SRE_CODE *end) case SRE_OP_CHARSET: offset = 32/sizeof(SRE_CODE); /* 32-byte bitmap */ - if (code+offset < code || code+offset > end) + if (offset > end-code) FAIL; code += offset; break; @@ -2825,7 +2833,7 @@ _validate_charset(SRE_CODE *code, SRE_CODE *end) case SRE_OP_BIGCHARSET: GET_ARG; /* Number of blocks */ offset = 256/sizeof(SRE_CODE); /* 256-byte table */ - if (code+offset < code || code+offset > end) + if (offset > end-code) FAIL; /* Make sure that each byte points to a valid block */ for (i = 0; i < 256; i++) { @@ -2834,7 +2842,7 @@ _validate_charset(SRE_CODE *code, SRE_CODE *end) } code += offset; offset = arg * 32/sizeof(SRE_CODE); /* 32-byte bitmap times arg */ - if (code+offset < code || code+offset > end) + if (offset > end-code) FAIL; code += offset; break; @@ -2985,11 +2993,11 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) GET_ARG; prefix_len = arg; GET_ARG; /* prefix skip */ /* Here comes the prefix string */ - if (code+prefix_len < code || code+prefix_len > newcode) + if (prefix_len > newcode-code) FAIL; code += prefix_len; /* And here comes the overlap table */ - if (code+prefix_len < code || code+prefix_len > newcode) + if (prefix_len > newcode-code) FAIL; /* Each overlap value should be < prefix_len */ for (i = 0; i < prefix_len; i++) { @@ -3118,7 +3126,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) to allow arbitrary jumps anywhere in the code; so we just look for a JUMP opcode preceding our skip target. */ - if (skip >= 3 && code+skip-3 >= code && + if (skip >= 3 && skip-3 < end-code && code[skip-3] == SRE_OP_JUMP) { VTRACE(("both then and else parts present\n")); diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 195e5b6491f..d95568cf167 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -18,6 +18,11 @@ #ifdef WITH_THREAD #include "pythread.h" + +#ifdef HAVE_PTHREAD_ATFORK +# include +#endif + #define PySSL_BEGIN_ALLOW_THREADS { \ PyThreadState *_save = NULL; \ if (_ssl_locks_count>0) {_save = PyEval_SaveThread();} @@ -281,6 +286,7 @@ newPySSLObject(PySocketSockObject *Sock, char *key_file, char *cert_file, self->ssl = NULL; self->ctx = NULL; self->Socket = NULL; + self->shutdown_seen_zero = 0; /* Make sure the SSL error state is initialized */ (void) ERR_get_state(); @@ -686,7 +692,7 @@ _get_peer_alt_names (X509 *certificate) { int i, j; PyObject *peer_alt_names = Py_None; - PyObject *v, *t; + PyObject *v = NULL, *t; X509_EXTENSION *ext = NULL; GENERAL_NAMES *names = NULL; GENERAL_NAME *name; @@ -738,13 +744,16 @@ _get_peer_alt_names (X509 *certificate) { ext->value->length)); for(j = 0; j < sk_GENERAL_NAME_num(names); j++) { - /* get a rendering of each name in the set of names */ + int gntype; + ASN1_STRING *as = NULL; name = sk_GENERAL_NAME_value(names, j); - if (name->type == GEN_DIRNAME) { - - /* we special-case DirName as a tuple of tuples of attributes */ + gntype = name->type; + switch (gntype) { + case GEN_DIRNAME: + /* we special-case DirName as a tuple of + tuples of attributes */ t = PyTuple_New(2); if (t == NULL) { @@ -764,11 +773,61 @@ _get_peer_alt_names (X509 *certificate) { goto fail; } PyTuple_SET_ITEM(t, 1, v); + break; - } else { + case GEN_EMAIL: + case GEN_DNS: + case GEN_URI: + /* GENERAL_NAME_print() doesn't handle NULL bytes in ASN1_string + correctly, CVE-2013-4238 */ + t = PyTuple_New(2); + if (t == NULL) + goto fail; + switch (gntype) { + case GEN_EMAIL: + v = PyString_FromString("email"); + as = name->d.rfc822Name; + break; + case GEN_DNS: + v = PyString_FromString("DNS"); + as = name->d.dNSName; + break; + case GEN_URI: + v = PyString_FromString("URI"); + as = name->d.uniformResourceIdentifier; + break; + } + if (v == NULL) { + Py_DECREF(t); + goto fail; + } + PyTuple_SET_ITEM(t, 0, v); + v = PyString_FromStringAndSize((char *)ASN1_STRING_data(as), + ASN1_STRING_length(as)); + if (v == NULL) { + Py_DECREF(t); + goto fail; + } + PyTuple_SET_ITEM(t, 1, v); + break; + default: /* for everything else, we use the OpenSSL print form */ - + switch (gntype) { + /* check for new general name type */ + case GEN_OTHERNAME: + case GEN_X400: + case GEN_EDIPARTY: + case GEN_IPADD: + case GEN_RID: + break; + default: + if (PyErr_Warn(PyExc_RuntimeWarning, + "Unknown general name type") == -1) { + goto fail; + } + break; + } (void) BIO_reset(biobuf); GENERAL_NAME_print(biobuf, name); len = BIO_gets(biobuf, buf, sizeof(buf)-1); @@ -794,6 +853,7 @@ _get_peer_alt_names (X509 *certificate) { goto fail; } PyTuple_SET_ITEM(t, 1, v); + break; } /* and add that rendering to the list */ @@ -1192,6 +1252,12 @@ static PyObject *PySSL_SSLwrite(PySSLObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "s*:write", &buf)) return NULL; + if (buf.len > INT_MAX) { + PyErr_Format(PyExc_OverflowError, + "string longer than %d bytes", INT_MAX); + goto error; + } + /* just in case the blocking state of the socket has been changed */ nonblocking = (self->Socket->sock_timeout >= 0.0); BIO_set_nbio(SSL_get_rbio(self->ssl), nonblocking); @@ -1213,7 +1279,7 @@ static PyObject *PySSL_SSLwrite(PySSLObject *self, PyObject *args) } do { PySSL_BEGIN_ALLOW_THREADS - len = SSL_write(self->ssl, buf.buf, buf.len); + len = SSL_write(self->ssl, buf.buf, (int)buf.len); err = SSL_get_error(self->ssl, len); PySSL_END_ALLOW_THREADS if (PyErr_CheckSignals()) { @@ -1395,7 +1461,7 @@ static PyObject *PySSL_SSLshutdown(PySSLObject *self) * Otherwise OpenSSL might read in too much data, * eating clear text data that happens to be * transmitted after the SSL shutdown. - * Should be safe to call repeatedly everytime this + * Should be safe to call repeatedly every time this * function is used and the shutdown_seen_zero != 0 * condition is met. */ @@ -1559,9 +1625,68 @@ PyDoc_STRVAR(PySSL_RAND_egd_doc, \n\ Queries the entropy gather daemon (EGD) on the socket named by 'path'.\n\ Returns number of bytes read. Raises SSLError if connection to EGD\n\ -fails or if it does provide enough data to seed PRNG."); +fails or if it does not provide enough data to seed PRNG."); + +/* Seed OpenSSL's PRNG at fork(), http://bugs.python.org/issue18747 + * + * The parent handler seeds the PRNG from pseudo-random data like pid, the + * current time (miliseconds or seconds) and an uninitialized array. + * The array contains stack variables that are impossible to predict + * on most systems, e.g. function return address (subject to ASLR), the + * stack protection canary and automatic variables. + * The code is inspired by Apache's ssl_rand_seed() function. + * + * Note: + * The code uses pthread_atfork() until Python has a proper atfork API. The + * handlers are not removed from the child process. A parent handler is used + * instead of a child handler because fork() is supposed to be async-signal + * safe but the handler calls unsafe functions. + */ + +#if defined(HAVE_PTHREAD_ATFORK) && defined(WITH_THREAD) +#define PYSSL_RAND_ATFORK 1 +static void +PySSL_RAND_atfork_parent(void) +{ + struct { + char stack[128]; /* uninitialized (!) stack data, 128 is an + arbitrary number. */ + pid_t pid; /* current pid */ + time_t time; /* current time */ + } seed; + +#ifdef WITH_VALGRIND + VALGRIND_MAKE_MEM_DEFINED(seed.stack, sizeof(seed.stack)); #endif + seed.pid = getpid(); + seed.time = time(NULL); + RAND_add((unsigned char *)&seed, sizeof(seed), 0.0); +} + +static int +PySSL_RAND_atfork(void) +{ + static int registered = 0; + int retval; + + if (registered) + return 0; + + retval = pthread_atfork(NULL, /* prepare */ + PySSL_RAND_atfork_parent, /* parent */ + NULL); /* child */ + if (retval != 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + registered = 1; + return 0; +} +#endif /* HAVE_PTHREAD_ATFORK */ + +#endif /* HAVE_OPENSSL_RAND */ + /* List of functions exported by this module. */ @@ -1589,9 +1714,21 @@ static PyMethodDef PySSL_methods[] = { static PyThread_type_lock *_ssl_locks = NULL; -static unsigned long _ssl_thread_id_function (void) { +#if OPENSSL_VERSION_NUMBER >= 0x10000000 +/* use new CRYPTO_THREADID API. */ +static void +_ssl_threadid_callback(CRYPTO_THREADID *id) +{ + CRYPTO_THREADID_set_numeric(id, + (unsigned long)PyThread_get_thread_ident()); +} +#else +/* deprecated CRYPTO_set_id_callback() API. */ +static unsigned long +_ssl_thread_id_function (void) { return PyThread_get_thread_ident(); } +#endif static void _ssl_thread_locking_function (int mode, int n, const char *file, int line) { /* this function is needed to perform locking on shared data @@ -1642,7 +1779,11 @@ static int _setup_ssl_threads(void) { } } CRYPTO_set_locking_callback(_ssl_thread_locking_function); +#if OPENSSL_VERSION_NUMBER >= 0x10000000 + CRYPTO_THREADID_set_callback(_ssl_threadid_callback); +#else CRYPTO_set_id_callback(_ssl_thread_id_function); +#endif } return 1; } @@ -1757,4 +1898,9 @@ init_ssl(void) r = PyString_FromString(SSLeay_version(SSLEAY_VERSION)); if (r == NULL || PyModule_AddObject(m, "OPENSSL_VERSION", r)) return; + +#ifdef PYSSL_RAND_ATFORK + if (PySSL_RAND_atfork() == -1) + return; +#endif } diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f1968e2a70c..4e7d47d8593 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -1118,7 +1118,7 @@ unicode_encodedecimal(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "u#|s", &unicode, &length, &errors)) return NULL; - decimal_length = length * 7; /* len('€') */ + decimal_length = length * 10; /* len('􏿿') */ decimal = PyBytes_FromStringAndSize(NULL, decimal_length); if (decimal == NULL) return NULL; @@ -1813,7 +1813,7 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) ; test_structmembers *ob; const char *s = NULL; - Py_ssize_t string_len = 0; + int string_len = 0; ob = PyObject_New(test_structmembers, type); if (ob == NULL) return NULL; diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 7872df3bf18..0627d60b677 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -47,6 +47,10 @@ Copyright (C) 1994 Steen Lumholt. #define PyBool_FromLong PyInt_FromLong #endif +#define CHECK_SIZE(size, elemsize) \ + ((size_t)(size) <= (size_t)INT_MAX && \ + (size_t)(size) <= UINT_MAX / (size_t)(elemsize)) + /* Starting with Tcl 8.4, many APIs offer const-correctness. Unfortunately, making _tkinter correct for this API means to break earlier versions. USE_COMPAT_CONST allows to make _tkinter work with both 8.4 and @@ -378,7 +382,7 @@ Merge(PyObject *args) char **argv = NULL; int fvStore[ARGSZ]; int *fv = NULL; - int argc = 0, fvc = 0, i; + Py_ssize_t argc = 0, fvc = 0, i; char *res = NULL; if (!(tmp = PyList_New(0))) @@ -400,8 +404,12 @@ Merge(PyObject *args) argc = PyTuple_Size(args); if (argc > ARGSZ) { - argv = (char **)ckalloc(argc * sizeof(char *)); - fv = (int *)ckalloc(argc * sizeof(int)); + if (!CHECK_SIZE(argc, sizeof(char *))) { + PyErr_SetString(PyExc_OverflowError, "tuple is too long"); + goto finally; + } + argv = (char **)ckalloc((size_t)argc * sizeof(char *)); + fv = (int *)ckalloc((size_t)argc * sizeof(int)); if (argv == NULL || fv == NULL) { PyErr_NoMemory(); goto finally; @@ -547,6 +555,33 @@ SplitObj(PyObject *arg) return Split(PyString_AsString(arg)); /* Fall through, returning arg. */ } + else if (PyUnicode_Check(arg)) { + int argc; + char **argv; + char *list; + PyObject *s = PyUnicode_AsUTF8String(arg); + + if (s == NULL) { + Py_INCREF(arg); + return arg; + } + list = PyString_AsString(s); + + if (list == NULL || + Tcl_SplitList((Tcl_Interp *)NULL, list, &argc, &argv) != TCL_OK) { + Py_DECREF(s); + Py_INCREF(arg); + return arg; + } + Tcl_Free(FREECAST argv); + if (argc > 1) { + PyObject *v = Split(list); + Py_DECREF(s); + return v; + } + Py_DECREF(s); + /* Fall through, returning arg. */ + } Py_INCREF(arg); return arg; } @@ -956,12 +991,18 @@ AsObj(PyObject *value) else if (PyFloat_Check(value)) return Tcl_NewDoubleObj(PyFloat_AS_DOUBLE(value)); else if (PyTuple_Check(value)) { - Tcl_Obj **argv = (Tcl_Obj**) - ckalloc(PyTuple_Size(value)*sizeof(Tcl_Obj*)); - int i; + Tcl_Obj **argv; + Py_ssize_t size, i; + + size = PyTuple_Size(value); + if (!CHECK_SIZE(size, sizeof(Tcl_Obj *))) { + PyErr_SetString(PyExc_OverflowError, "tuple is too long"); + return NULL; + } + argv = (Tcl_Obj **) ckalloc(((size_t)size) * sizeof(Tcl_Obj *)); if(!argv) return 0; - for(i=0;i= size) outbuf = (Tcl_UniChar*)ckalloc(allocsize); /* Else overflow occurred, and we take the next exit */ @@ -1171,7 +1217,7 @@ static Tcl_Obj** Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, int *pobjc) { Tcl_Obj **objv = objStore; - int objc = 0, i; + Py_ssize_t objc = 0, i; if (args == NULL) /* do nothing */; @@ -1186,7 +1232,11 @@ Tkapp_CallArgs(PyObject *args, Tcl_Obj** objStore, int *pobjc) objc = PyTuple_Size(args); if (objc > ARGSZ) { - objv = (Tcl_Obj **)ckalloc(objc * sizeof(char *)); + if (!CHECK_SIZE(objc, sizeof(Tcl_Obj *))) { + PyErr_SetString(PyExc_OverflowError, "tuple is too long"); + return NULL; + } + objv = (Tcl_Obj **)ckalloc(((size_t)objc) * sizeof(Tcl_Obj *)); if (objv == NULL) { PyErr_NoMemory(); objc = 0; @@ -1954,16 +2004,35 @@ Tkapp_SplitList(PyObject *self, PyObject *args) char *list; int argc; char **argv; - PyObject *v; + PyObject *arg, *v; int i; - if (PyTuple_Size(args) == 1) { - v = PyTuple_GetItem(args, 0); - if (PyTuple_Check(v)) { - Py_INCREF(v); - return v; + if (!PyArg_ParseTuple(args, "O:splitlist", &arg)) + return NULL; + if (PyTclObject_Check(arg)) { + int objc; + Tcl_Obj **objv; + if (Tcl_ListObjGetElements(Tkapp_Interp(self), + ((PyTclObject*)arg)->value, + &objc, &objv) == TCL_ERROR) { + return Tkinter_Error(self); + } + if (!(v = PyTuple_New(objc))) + return NULL; + for (i = 0; i < objc; i++) { + PyObject *s = FromObj(self, objv[i]); + if (!s || PyTuple_SetItem(v, i, s)) { + Py_DECREF(v); + return NULL; + } } + return v; } + if (PyTuple_Check(arg)) { + Py_INCREF(arg); + return arg; + } + if (!PyArg_ParseTuple(args, "et:splitlist", "utf-8", &list)) return NULL; @@ -1994,16 +2063,38 @@ Tkapp_SplitList(PyObject *self, PyObject *args) static PyObject * Tkapp_Split(PyObject *self, PyObject *args) { - PyObject *v; + PyObject *arg, *v; char *list; - if (PyTuple_Size(args) == 1) { - PyObject* o = PyTuple_GetItem(args, 0); - if (PyTuple_Check(o)) { - o = SplitObj(o); - return o; + if (!PyArg_ParseTuple(args, "O:split", &arg)) + return NULL; + if (PyTclObject_Check(arg)) { + Tcl_Obj *value = ((PyTclObject*)arg)->value; + int objc; + Tcl_Obj **objv; + int i; + if (Tcl_ListObjGetElements(Tkapp_Interp(self), value, + &objc, &objv) == TCL_ERROR) { + return FromObj(self, value); } + if (objc == 0) + return PyString_FromString(""); + if (objc == 1) + return FromObj(self, objv[0]); + if (!(v = PyTuple_New(objc))) + return NULL; + for (i = 0; i < objc; i++) { + PyObject *s = FromObj(self, objv[i]); + if (!s || PyTuple_SetItem(v, i, s)) { + Py_DECREF(v); + return NULL; + } + } + return v; } + if (PyTuple_Check(arg)) + return SplitObj(arg); + if (!PyArg_ParseTuple(args, "et:split", "utf-8", &list)) return NULL; v = Split(list); @@ -2723,7 +2814,7 @@ Tkapp_InterpAddr(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, ":interpaddr")) return NULL; - return PyInt_FromLong((long)Tkapp_Interp(self)); + return PyLong_FromVoidPtr(Tkapp_Interp(self)); } static PyObject * diff --git a/Modules/cPickle.c b/Modules/cPickle.c index d74ec5b7bba..8145bbf381a 100644 --- a/Modules/cPickle.c +++ b/Modules/cPickle.c @@ -3643,17 +3643,19 @@ load_string(Unpicklerobject *self) /* Strip outermost quotes */ - while (s[len-1] <= ' ') + while (len > 0 && s[len-1] <= ' ') len--; - if(s[0]=='"' && s[len-1]=='"'){ + if (len > 1 && s[0]=='"' && s[len-1]=='"') { s[len-1] = '\0'; p = s + 1 ; len -= 2; - } else if(s[0]=='\'' && s[len-1]=='\''){ + } + else if (len > 1 && s[0]=='\'' && s[len-1]=='\'') { s[len-1] = '\0'; p = s + 1 ; len -= 2; - } else + } + else goto insecure; /********************************************/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 0dc6bdb3968..57206785e71 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -1006,6 +1006,13 @@ cmath_rect(PyObject *self, PyObject *args) else errno = 0; } + else if (phi == 0.0) { + /* Workaround for buggy results with phi=-0.0 on OS X 10.8. See + bugs.python.org/issue18513. */ + z.real = r; + z.imag = r * phi; + errno = 0; + } else { z.real = r * cos(phi); z.imag = r * sin(phi); diff --git a/Modules/dbmmodule.c b/Modules/dbmmodule.c index f9c99a8b8f0..8b16def2676 100644 --- a/Modules/dbmmodule.c +++ b/Modules/dbmmodule.c @@ -168,11 +168,13 @@ static int dbm_contains(register dbmobject *dp, PyObject *v) { datum key, val; + char *ptr; + Py_ssize_t size; - if (PyString_AsStringAndSize(v, (char **)&key.dptr, - (Py_ssize_t *)&key.dsize)) { + if (PyString_AsStringAndSize(v, &ptr, &size)) return -1; - } + key.dptr = ptr; + key.dsize = size; /* Expand check_dbmobject_open to return -1 */ if (dp->di_dbm == NULL) { diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 997867a136e..cd5c98f1813 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -299,7 +299,7 @@ PyDoc_STRVAR(flock_doc, "flock(fd, operation)\n\ \n\ Perform the lock operation op on file descriptor fd. See the Unix \n\ -manual page for flock(3) for details. (On some systems, this function is\n\ +manual page for flock(2) for details. (On some systems, this function is\n\ emulated using fcntl().)"); diff --git a/Modules/getpath.c b/Modules/getpath.c index 9faafa3555d..de96d474d21 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -335,12 +335,27 @@ search_for_exec_prefix(char *argv0_path, char *home) return 1; } - /* Check to see if argv[0] is in the build directory */ + /* Check to see if argv[0] is in the build directory. "pybuilddir.txt" + is written by setup.py and contains the relative path to the location + of shared library modules. */ strcpy(exec_prefix, argv0_path); - joinpath(exec_prefix, "Modules/Setup"); + joinpath(exec_prefix, "pybuilddir.txt"); if (isfile(exec_prefix)) { - reduce(exec_prefix); - return -1; + FILE *f = fopen(exec_prefix, "r"); + if (f == NULL) + errno = 0; + else { + char rel_builddir_path[MAXPATHLEN+1]; + size_t n; + n = fread(rel_builddir_path, 1, MAXPATHLEN, f); + rel_builddir_path[n] = '\0'; + fclose(f); + if (n >= 0) { + strcpy(exec_prefix, argv0_path); + joinpath(exec_prefix, rel_builddir_path); + return -1; + } + } } /* Search from argv0_path, until root is found */ diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index 55da9b8aa28..040d8b0b100 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -11,7 +11,7 @@ static PyStructSequence_Field struct_group_type_fields[] = { {"gr_name", "group name"}, {"gr_passwd", "password"}, {"gr_gid", "group id"}, - {"gr_mem", "group memebers"}, + {"gr_mem", "group members"}, {0} }; diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index ea822913662..c0456b839f7 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -3426,10 +3426,10 @@ PyDoc_STRVAR(count_doc, Return a count object whose .next() method returns consecutive values.\n\ Equivalent to:\n\n\ def count(firstval=0, step=1):\n\ - x = firstval\n\ - while 1:\n\ - yield x\n\ - x += step\n"); + x = firstval\n\ + while 1:\n\ + yield x\n\ + x += step\n"); static PyTypeObject count_type = { PyVarObject_HEAD_INIT(NULL, 0) diff --git a/Modules/operator.c b/Modules/operator.c index 274d8aad39f..3f005884426 100644 --- a/Modules/operator.c +++ b/Modules/operator.c @@ -412,8 +412,8 @@ PyDoc_STRVAR(itemgetter_doc, "itemgetter(item, ...) --> itemgetter object\n\ \n\ Return a callable object that fetches the given item(s) from its operand.\n\ -After, f=itemgetter(2), the call f(r) returns r[2].\n\ -After, g=itemgetter(2,5,3), the call g(r) returns (r[2], r[5], r[3])"); +After f = itemgetter(2), the call f(r) returns r[2].\n\ +After g = itemgetter(2, 5, 3), the call g(r) returns (r[2], r[5], r[3])"); static PyTypeObject itemgetter_type = { PyVarObject_HEAD_INIT(NULL, 0) @@ -592,9 +592,9 @@ PyDoc_STRVAR(attrgetter_doc, "attrgetter(attr, ...) --> attrgetter object\n\ \n\ Return a callable object that fetches the given attribute(s) from its operand.\n\ -After, f=attrgetter('name'), the call f(r) returns r.name.\n\ -After, g=attrgetter('name', 'date'), the call g(r) returns (r.name, r.date).\n\ -After, h=attrgetter('name.first', 'name.last'), the call h(r) returns\n\ +After f = attrgetter('name'), the call f(r) returns r.name.\n\ +After g = attrgetter('name', 'date'), the call g(r) returns (r.name, r.date).\n\ +After h = attrgetter('name.first', 'name.last'), the call h(r) returns\n\ (r.name.first, r.name.last)."); static PyTypeObject attrgetter_type = { @@ -724,8 +724,8 @@ PyDoc_STRVAR(methodcaller_doc, "methodcaller(name, ...) --> methodcaller object\n\ \n\ Return a callable object that calls the given method on its operand.\n\ -After, f = methodcaller('name'), the call f(r) returns r.name().\n\ -After, g = methodcaller('name', 'date', foo=1), the call g(r) returns\n\ +After f = methodcaller('name'), the call f(r) returns r.name().\n\ +After g = methodcaller('name', 'date', foo=1), the call g(r) returns\n\ r.name('date', foo=1)."); static PyTypeObject methodcaller_type = { diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 65ade9fb9fd..8352dd3e00a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4054,6 +4054,34 @@ posix_getgroups(PyObject *self, PyObject *noargs) gid_t* alt_grouplist = grouplist; int n; +#ifdef __APPLE__ + /* Issue #17557: As of OS X 10.8, getgroups(2) no longer raises EINVAL if + * there are more groups than can fit in grouplist. Therefore, on OS X + * always first call getgroups with length 0 to get the actual number + * of groups. + */ + n = getgroups(0, NULL); + if (n < 0) { + return posix_error(); + } else if (n <= MAX_GROUPS) { + /* groups will fit in existing array */ + alt_grouplist = grouplist; + } else { + alt_grouplist = PyMem_Malloc(n * sizeof(gid_t)); + if (alt_grouplist == NULL) { + errno = EINVAL; + return posix_error(); + } + } + + n = getgroups(n, alt_grouplist); + if (n == -1) { + if (alt_grouplist != grouplist) { + PyMem_Free(alt_grouplist); + } + return posix_error(); + } +#else n = getgroups(MAX_GROUPS, grouplist); if (n < 0) { if (errno == EINVAL) { @@ -4080,6 +4108,8 @@ posix_getgroups(PyObject *self, PyObject *noargs) return posix_error(); } } +#endif + result = PyList_New(n); if (result != NULL) { int i; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index f2691136ca8..8de3fb17d33 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1252,6 +1252,13 @@ PyUnknownEncodingHandler(void *encodingHandlerData, if (_u_string == NULL) return result; + if (PyUnicode_GET_SIZE(_u_string) != 256) { + Py_DECREF(_u_string); + PyErr_SetString(PyExc_ValueError, + "multi-byte encodings are not supported"); + return result; + } + for (i = 0; i < 256; i++) { /* Stupid to access directly, but fast */ Py_UNICODE c = _u_string->str[i]; diff --git a/Modules/readline.c b/Modules/readline.c index b5e258db67e..233ebf95ad1 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -69,6 +69,10 @@ on_completion_display_matches_hook(char **matches, int num_matches, int max_length); +/* Memory allocated for rl_completer_word_break_characters + (see issue #17289 for the motivation). */ +static char *completer_word_break_characters; + /* Exported function to send one line to readline's init file parser */ static PyObject * @@ -344,12 +348,20 @@ set_completer_delims(PyObject *self, PyObject *args) { char *break_chars; - if(!PyArg_ParseTuple(args, "s:set_completer_delims", &break_chars)) { + if (!PyArg_ParseTuple(args, "s:set_completer_delims", &break_chars)) { return NULL; } - free((void*)rl_completer_word_break_characters); - rl_completer_word_break_characters = strdup(break_chars); - Py_RETURN_NONE; + /* Keep a reference to the allocated memory in the module state in case + some other module modifies rl_completer_word_break_characters + (see issue #17289). */ + free(completer_word_break_characters); + completer_word_break_characters = strdup(break_chars); + if (completer_word_break_characters) { + rl_completer_word_break_characters = completer_word_break_characters; + Py_RETURN_NONE; + } + else + return PyErr_NoMemory(); } PyDoc_STRVAR(doc_set_completer_delims, @@ -893,7 +905,8 @@ setup_readline(void) /* Set our completion function */ rl_attempted_completion_function = (CPPFunction *)flex_complete; /* Set Python word break characters */ - rl_completer_word_break_characters = + completer_word_break_characters = + rl_completer_word_break_characters = strdup(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?"); /* All nonalphanums except '.' */ @@ -906,7 +919,7 @@ setup_readline(void) */ #ifdef __APPLE__ if (using_libedit_emulation) - rl_read_init_file(NULL); + rl_read_init_file(NULL); else #endif /* __APPLE__ */ rl_initialize(); @@ -1137,8 +1150,6 @@ initreadline(void) if (m == NULL) return; - - PyOS_ReadlineFunctionPointer = call_readline; setup_readline(); } diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 650d9fe3fba..6a3652160a9 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -318,6 +318,7 @@ typedef struct { int ufd_uptodate; int ufd_len; struct pollfd *ufds; + int poll_running; } pollObject; static PyTypeObject poll_Type; @@ -513,16 +514,27 @@ poll_poll(pollObject *self, PyObject *args) return NULL; } + /* Avoid concurrent poll() invocation, issue 8865 */ + if (self->poll_running) { + PyErr_SetString(PyExc_RuntimeError, + "concurrent poll() invocation"); + return NULL; + } + /* Ensure the ufd array is up to date */ if (!self->ufd_uptodate) if (update_ufd_array(self) == 0) return NULL; + self->poll_running = 1; + /* call poll() */ Py_BEGIN_ALLOW_THREADS poll_result = poll(self->ufds, self->ufd_len, timeout); Py_END_ALLOW_THREADS + self->poll_running = 0; + if (poll_result < 0) { PyErr_SetFromErrno(SelectError); return NULL; @@ -599,6 +611,7 @@ newPollObject(void) array pointed to by ufds matches the contents of the dictionary. */ self->ufd_uptodate = 0; self->ufds = NULL; + self->poll_running = 0; self->dict = PyDict_New(); if (self->dict == NULL) { Py_DECREF(self); @@ -1203,6 +1216,23 @@ static PyTypeObject kqueue_queue_Type; # error uintptr_t does not match int, long, or long long! #endif +/* + * kevent is not standard and its members vary across BSDs. + */ +#if !defined(__OpenBSD__) +# define IDENT_TYPE T_UINTPTRT +# define IDENT_CAST Py_intptr_t +# define DATA_TYPE T_INTPTRT +# define DATA_FMT_UNIT INTPTRT_FMT_UNIT +# define IDENT_AsType PyLong_AsUintptr_t +#else +# define IDENT_TYPE T_UINT +# define IDENT_CAST int +# define DATA_TYPE T_INT +# define DATA_FMT_UNIT "i" +# define IDENT_AsType PyLong_AsUnsignedLong +#endif + /* Unfortunately, we can't store python objects in udata, because * kevents in the kernel can be removed without warning, which would * forever lose the refcount on the object stored with it. @@ -1210,11 +1240,11 @@ static PyTypeObject kqueue_queue_Type; #define KQ_OFF(x) offsetof(kqueue_event_Object, x) static struct PyMemberDef kqueue_event_members[] = { - {"ident", T_UINTPTRT, KQ_OFF(e.ident)}, + {"ident", IDENT_TYPE, KQ_OFF(e.ident)}, {"filter", T_SHORT, KQ_OFF(e.filter)}, {"flags", T_USHORT, KQ_OFF(e.flags)}, {"fflags", T_UINT, KQ_OFF(e.fflags)}, - {"data", T_INTPTRT, KQ_OFF(e.data)}, + {"data", DATA_TYPE, KQ_OFF(e.data)}, {"udata", T_UINTPTRT, KQ_OFF(e.udata)}, {NULL} /* Sentinel */ }; @@ -1240,7 +1270,7 @@ kqueue_event_init(kqueue_event_Object *self, PyObject *args, PyObject *kwds) PyObject *pfd; static char *kwlist[] = {"ident", "filter", "flags", "fflags", "data", "udata", NULL}; - static char *fmt = "O|hhi" INTPTRT_FMT_UNIT UINTPTRT_FMT_UNIT ":kevent"; + static char *fmt = "O|hHI" DATA_FMT_UNIT UINTPTRT_FMT_UNIT ":kevent"; EV_SET(&(self->e), 0, EVFILT_READ, EV_ADD, 0, 0, 0); /* defaults */ @@ -1250,8 +1280,12 @@ kqueue_event_init(kqueue_event_Object *self, PyObject *args, PyObject *kwds) return -1; } - if (PyLong_Check(pfd)) { - self->e.ident = PyLong_AsUintptr_t(pfd); + if (PyLong_Check(pfd) +#if IDENT_TYPE == T_UINT + && PyLong_AsUnsignedLong(pfd) <= UINT_MAX +#endif + ) { + self->e.ident = IDENT_AsType(pfd); } else { self->e.ident = PyObject_AsFileDescriptor(pfd); @@ -1279,10 +1313,10 @@ kqueue_event_richcompare(kqueue_event_Object *s, kqueue_event_Object *o, Py_TYPE(s)->tp_name, Py_TYPE(o)->tp_name); return NULL; } - if (((result = s->e.ident - o->e.ident) == 0) && + if (((result = (IDENT_CAST)(s->e.ident - o->e.ident)) == 0) && ((result = s->e.filter - o->e.filter) == 0) && ((result = s->e.flags - o->e.flags) == 0) && - ((result = s->e.fflags - o->e.fflags) == 0) && + ((result = (int)(s->e.fflags - o->e.fflags)) == 0) && ((result = s->e.data - o->e.data) == 0) && ((result = s->e.udata - o->e.udata) == 0) ) { diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index d5d628328b1..184b3a91a39 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -321,7 +321,10 @@ signal_signal(PyObject *self, PyObject *args) Handlers[sig_num].tripped = 0; Py_INCREF(obj); Handlers[sig_num].func = obj; - return old_handler; + if (old_handler != NULL) + return old_handler; + else + Py_RETURN_NONE; } PyDoc_STRVAR(signal_doc, @@ -349,8 +352,13 @@ signal_getsignal(PyObject *self, PyObject *args) return NULL; } old_handler = Handlers[sig_num].func; - Py_INCREF(old_handler); - return old_handler; + if (old_handler != NULL) { + Py_INCREF(old_handler); + return old_handler; + } + else { + Py_RETURN_NONE; + } } PyDoc_STRVAR(getsignal_doc, diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index bdc055dd384..2735ecc7735 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -4179,6 +4179,15 @@ socket_getaddrinfo(PyObject *self, PyObject *args) "getaddrinfo() argument 2 must be integer or string"); goto err; } +#if defined(__APPLE__) && defined(AI_NUMERICSERV) + if ((flags & AI_NUMERICSERV) && (pptr == NULL || (pptr[0] == '0' && pptr[1] == 0))) { + /* On OSX upto at least OSX 10.8 getaddrinfo crashes + * if AI_NUMERICSERV is set and the servname is NULL or "0". + * This workaround avoids a segfault in libsystem. + */ + pptr = "00"; + } +#endif memset(&hints, 0, sizeof(hints)); hints.ai_family = family; hints.ai_socktype = socktype; diff --git a/Modules/sre.h b/Modules/sre.h index 200e492d264..22a2d5dd96c 100644 --- a/Modules/sre.h +++ b/Modules/sre.h @@ -20,14 +20,14 @@ # if SIZEOF_SIZE_T > 4 # define SRE_MAXREPEAT (~(SRE_CODE)0) # else -# define SRE_MAXREPEAT ((SRE_CODE)PY_SSIZE_T_MAX + 1u) +# define SRE_MAXREPEAT ((SRE_CODE)PY_SSIZE_T_MAX) # endif #else -# define SRE_CODE unsigned long -# if SIZEOF_SIZE_T > SIZEOF_LONG +# define SRE_CODE unsigned int +# if SIZEOF_SIZE_T > SIZEOF_INT # define SRE_MAXREPEAT (~(SRE_CODE)0) # else -# define SRE_MAXREPEAT ((SRE_CODE)PY_SSIZE_T_MAX + 1u) +# define SRE_MAXREPEAT ((SRE_CODE)PY_SSIZE_T_MAX) # endif #endif diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 95bdf3cfd1e..6f9c7e8e831 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -506,7 +506,7 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) stackptr = 0; isize = PyUnicode_GET_SIZE(input); - /* Overallocate atmost 10 characters. */ + /* Overallocate at most 10 characters. */ space = (isize > 10 ? 10 : isize) + isize; result = PyUnicode_FromUnicode(NULL, space); if (!result) @@ -520,7 +520,7 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) while(stackptr) { Py_UNICODE code = stack[--stackptr]; /* Hangul Decomposition adds three characters in - a single step, so we need atleast that much room. */ + a single step, so we need at least that much room. */ if (space < 3) { Py_ssize_t newsize = PyString_GET_SIZE(result) + 10; space += 10; diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 677ac0ebe5a..5ee0c15e3d7 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -885,7 +885,7 @@ complex_conjugate(PyObject *self) PyDoc_STRVAR(complex_conjugate_doc, "complex.conjugate() -> complex\n" "\n" -"Returns the complex conjugate of its argument. (3-4j).conjugate() == 3+4j."); +"Return the complex conjugate of its argument. (3-4j).conjugate() == 3+4j."); static PyObject * complex_getnewargs(PyComplexObject *v) @@ -897,7 +897,7 @@ complex_getnewargs(PyComplexObject *v) PyDoc_STRVAR(complex__format__doc, "complex.__format__() -> str\n" "\n" -"Converts to a string according to format_spec."); +"Convert to a string according to format_spec."); static PyObject * complex__format__(PyObject* self, PyObject* args) diff --git a/Objects/dictobject.c b/Objects/dictobject.c index ba36b180cad..39e70359010 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -2919,6 +2919,10 @@ dictview_repr(dictviewobject *dv) return NULL; seq_str = PyObject_Repr(seq); + if (seq_str == NULL) { + Py_DECREF(seq); + return NULL; + } result = PyString_FromFormat("%s(%s)", Py_TYPE(dv)->tp_name, PyString_AS_STRING(seq_str)); Py_DECREF(seq_str); diff --git a/Objects/floatobject.c b/Objects/floatobject.c index ba867ef149f..2bec0fbc6ff 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1783,9 +1783,9 @@ float_as_integer_ratio(PyObject *v, PyObject *unused) PyDoc_STRVAR(float_as_integer_ratio_doc, "float.as_integer_ratio() -> (int, int)\n" "\n" -"Returns a pair of integers, whose ratio is exactly equal to the original\n" +"Return a pair of integers, whose ratio is exactly equal to the original\n" "float and with a positive denominator.\n" -"Raises OverflowError on infinities and a ValueError on NaNs.\n" +"Raise OverflowError on infinities and a ValueError on NaNs.\n" "\n" ">>> (10.0).as_integer_ratio()\n" "(10, 1)\n" @@ -1970,7 +1970,7 @@ PyDoc_STRVAR(float_setformat_doc, "'IEEE, big-endian' or 'IEEE, little-endian', and in addition can only be\n" "one of the latter two if it appears to match the underlying C reality.\n" "\n" -"Overrides the automatic determination of C-level floating point type.\n" +"Override the automatic determination of C-level floating point type.\n" "This affects how floats are converted to and from binary strings."); static PyObject * @@ -2017,9 +2017,9 @@ PyDoc_STRVAR(float__format__doc, static PyMethodDef float_methods[] = { {"conjugate", (PyCFunction)float_float, METH_NOARGS, - "Returns self, the complex conjugate of any float."}, + "Return self, the complex conjugate of any float."}, {"__trunc__", (PyCFunction)float_trunc, METH_NOARGS, - "Returns the Integral closest to x between 0 and x."}, + "Return the Integral closest to x between 0 and x."}, {"as_integer_ratio", (PyCFunction)float_as_integer_ratio, METH_NOARGS, float_as_integer_ratio_doc}, {"fromhex", (PyCFunction)float_fromhex, @@ -2027,14 +2027,14 @@ static PyMethodDef float_methods[] = { {"hex", (PyCFunction)float_hex, METH_NOARGS, float_hex_doc}, {"is_integer", (PyCFunction)float_is_integer, METH_NOARGS, - "Returns True if the float is an integer."}, + "Return True if the float is an integer."}, #if 0 {"is_inf", (PyCFunction)float_is_inf, METH_NOARGS, - "Returns True if the float is positive or negative infinite."}, + "Return True if the float is positive or negative infinite."}, {"is_finite", (PyCFunction)float_is_finite, METH_NOARGS, - "Returns True if the float is finite, neither infinite nor NaN."}, + "Return True if the float is finite, neither infinite nor NaN."}, {"is_nan", (PyCFunction)float_is_nan, METH_NOARGS, - "Returns True if the float is not a number (NaN)."}, + "Return True if the float is not a number (NaN)."}, #endif {"__getnewargs__", (PyCFunction)float_getnewargs, METH_NOARGS}, {"__getformat__", (PyCFunction)float_getformat, diff --git a/Objects/listsort.txt b/Objects/listsort.txt index 31a5445c0e3..08fef96bdf0 100644 --- a/Objects/listsort.txt +++ b/Objects/listsort.txt @@ -60,6 +60,7 @@ Comparison with Python's Samplesort Hybrid /sort: ascending data 3sort: ascending, then 3 random exchanges +sort: ascending, then 10 random at the end + %sort: ascending, then randomly replace 1% of elements w/ random values ~sort: many duplicates =sort: all equal !sort: worst case scenario @@ -99,11 +100,13 @@ Comparison with Python's Samplesort Hybrid The algorithms are effectively identical in these cases, except that timsort does one less compare in \sort. - Now for the more interesting cases. lg(n!) is the information-theoretic - limit for the best any comparison-based sorting algorithm can do on - average (across all permutations). When a method gets significantly - below that, it's either astronomically lucky, or is finding exploitable - structure in the data. + Now for the more interesting cases. Where lg(x) is the logarithm of x to + the base 2 (e.g., lg(8)=3), lg(n!) is the information-theoretic limit for + the best any comparison-based sorting algorithm can do on average (across + all permutations). When a method gets significantly below that, it's + either astronomically lucky, or is finding exploitable structure in the + data. + n lg(n!) *sort 3sort +sort %sort ~sort !sort ------- ------- ------ ------- ------- ------ ------- -------- @@ -250,7 +253,7 @@ Computing minrun ---------------- If N < 64, minrun is N. IOW, binary insertion sort is used for the whole array then; it's hard to beat that given the overheads of trying something -fancier. +fancier (see note BINSORT). When N is a power of 2, testing on random data showed that minrun values of 16, 32, 64 and 128 worked about equally well. At 256 the data-movement cost @@ -378,10 +381,10 @@ with wildly unbalanced run lengths. Merge Memory ------------ -Merging adjacent runs of lengths A and B in-place is very difficult. -Theoretical constructions are known that can do it, but they're too difficult -and slow for practical use. But if we have temp memory equal to min(A, B), -it's easy. +Merging adjacent runs of lengths A and B in-place, and in linear time, is +difficult. Theoretical constructions are known that can do it, but they're +too difficult and slow for practical use. But if we have temp memory equal +to min(A, B), it's easy. If A is smaller (function merge_lo), copy A to a temp array, leave B alone, and then we can do the obvious merge algorithm left to right, from the temp @@ -456,10 +459,10 @@ finding the right spot early in B (more on that later). After finding such a k, the region of uncertainty is reduced to 2**(k-1) - 1 consecutive elements, and a straight binary search requires exactly k-1 -additional comparisons to nail it. Then we copy all the B's up to that -point in one chunk, and then copy A[0]. Note that no matter where A[0] -belongs in B, the combination of galloping + binary search finds it in no -more than about 2*lg(B) comparisons. +additional comparisons to nail it (see note REGION OF UNCERTAINTY). Then we +copy all the B's up to that point in one chunk, and then copy A[0]. Note +that no matter where A[0] belongs in B, the combination of galloping + binary +search finds it in no more than about 2*lg(B) comparisons. If we did a straight binary search, we could find it in no more than ceiling(lg(B+1)) comparisons -- but straight binary search takes that many @@ -572,11 +575,11 @@ Galloping Complication The description above was for merge_lo. merge_hi has to merge "from the other end", and really needs to gallop starting at the last element in a run instead of the first. Galloping from the first still works, but does more -comparisons than it should (this is significant -- I timed it both ways). -For this reason, the gallop_left() and gallop_right() functions have a -"hint" argument, which is the index at which galloping should begin. So -galloping can actually start at any index, and proceed at offsets of 1, 3, -7, 15, ... or -1, -3, -7, -15, ... from the starting index. +comparisons than it should (this is significant -- I timed it both ways). For +this reason, the gallop_left() and gallop_right() (see note LEFT OR RIGHT) +functions have a "hint" argument, which is the index at which galloping +should begin. So galloping can actually start at any index, and proceed at +offsets of 1, 3, 7, 15, ... or -1, -3, -7, -15, ... from the starting index. In the code as I type it's always called with either 0 or n-1 (where n is the # of elements in a run). It's tempting to try to do something fancier, @@ -675,3 +678,78 @@ immediately. The consequence is that it ends up using two compares to sort [2, 1]. Gratifyingly, timsort doesn't do any special-casing, so had to be taught how to deal with mixtures of ascending and descending runs efficiently in all cases. + + +NOTES +----- + +BINSORT +A "binary insertion sort" is just like a textbook insertion sort, but instead +of locating the correct position of the next item via linear (one at a time) +search, an equivalent to Python's bisect.bisect_right is used to find the +correct position in logarithmic time. Most texts don't mention this +variation, and those that do usually say it's not worth the bother: insertion +sort remains quadratic (expected and worst cases) either way. Speeding the +search doesn't reduce the quadratic data movement costs. + +But in CPython's case, comparisons are extraordinarily expensive compared to +moving data, and the details matter. Moving objects is just copying +pointers. Comparisons can be arbitrarily expensive (can invoke arbitary +user-supplied Python code), but even in simple cases (like 3 < 4) _all_ +decisions are made at runtime: what's the type of the left comparand? the +type of the right? do they need to be coerced to a common type? where's the +code to compare these types? And so on. Even the simplest Python comparison +triggers a large pile of C-level pointer dereferences, conditionals, and +function calls. + +So cutting the number of compares is almost always measurably helpful in +CPython, and the savings swamp the quadratic-time data movement costs for +reasonable minrun values. + + +LEFT OR RIGHT +gallop_left() and gallop_right() are akin to the Python bisect module's +bisect_left() and bisect_right(): they're the same unless the slice they're +searching contains a (at least one) value equal to the value being searched +for. In that case, gallop_left() returns the position immediately before the +leftmost equal value, and gallop_right() the position immediately after the +rightmost equal value. The distinction is needed to preserve stability. In +general, when merging adjacent runs A and B, gallop_left is used to search +thru B for where an element from A belongs, and gallop_right to search thru A +for where an element from B belongs. + + +REGION OF UNCERTAINTY +Two kinds of confusion seem to be common about the claim that after finding +a k such that + + B[2**(k-1) - 1] < A[0] <= B[2**k - 1] + +then a binary search requires exactly k-1 tries to find A[0]'s proper +location. For concreteness, say k=3, so B[3] < A[0] <= B[7]. + +The first confusion takes the form "OK, then the region of uncertainty is at +indices 3, 4, 5, 6 and 7: that's 5 elements, not the claimed 2**(k-1) - 1 = +3"; or the region is viewed as a Python slice and the objection is "but that's +the slice B[3:7], so has 7-3 = 4 elements". Resolution: we've already +compared A[0] against B[3] and against B[7], so A[0]'s correct location is +already known wrt _both_ endpoints. What remains is to find A[0]'s correct +location wrt B[4], B[5] and B[6], which spans 3 elements. Or in general, the +slice (leaving off both endpoints) (2**(k-1)-1)+1 through (2**k-1)-1 +inclusive = 2**(k-1) through (2**k-1)-1 inclusive, which has + (2**k-1)-1 - 2**(k-1) + 1 = + 2**k-1 - 2**(k-1) = + 2*2**k-1 - 2**(k-1) = + (2-1)*2**(k-1) - 1 = + 2**(k-1) - 1 +elements. + +The second confusion: "k-1 = 2 binary searches can find the correct location +among 2**(k-1) = 4 elements, but you're only applying it to 3 elements: we +could make this more efficient by arranging for the region of uncertainty to +span 2**(k-1) elements." Resolution: that confuses "elements" with +"locations". In a slice with N elements, there are N+1 _locations_. In the +example, with the region of uncertainty B[4], B[5], B[6], there are 4 +locations: before B[4], between B[4] and B[5], between B[5] and B[6], and +after B[6]. In general, across 2**(k-1)-1 elements, there are 2**(k-1) +locations. That's why k-1 binary searches are necessary and sufficient. diff --git a/Objects/longobject.c b/Objects/longobject.c index fb740dc4d6c..405be2e045a 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -3474,10 +3474,16 @@ long_pow(PyObject *v, PyObject *w, PyObject *x) goto Done; } - /* if base < 0: - base = base % modulus - Having the base positive just makes things easier. */ - if (Py_SIZE(a) < 0) { + /* Reduce base by modulus in some cases: + 1. If base < 0. Forcing the base non-negative makes things easier. + 2. If base is obviously larger than the modulus. The "small + exponent" case later can multiply directly by base repeatedly, + while the "large exponent" case multiplies directly by base 31 + times. It can be unboundedly faster to multiply by + base % modulus instead. + We could _always_ do this reduction, but l_divmod() isn't cheap, + so we only do it when it buys something. */ + if (Py_SIZE(a) < 0 || Py_SIZE(a) > Py_SIZE(c)) { if (l_divmod(a, c, NULL, &temp) < 0) goto Error; Py_DECREF(a); diff --git a/Objects/setobject.c b/Objects/setobject.c index af1ce16bc29..db5cee28403 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -212,7 +212,6 @@ static int set_insert_key(register PySetObject *so, PyObject *key, long hash) { register setentry *entry; - typedef setentry *(*lookupfunc)(PySetObject *, PyObject *, long); assert(so->lookup != NULL); entry = so->lookup(so, key, hash); diff --git a/Objects/stringlib/formatter.h b/Objects/stringlib/formatter.h index 6b282249b17..fd227511f36 100644 --- a/Objects/stringlib/formatter.h +++ b/Objects/stringlib/formatter.h @@ -928,7 +928,7 @@ format_float_internal(PyObject *value, Py_ssize_t n_total; int has_decimal; double val; - Py_ssize_t precision = format->precision; + Py_ssize_t precision; Py_ssize_t default_precision = 6; STRINGLIB_CHAR type = format->type; int add_pct = 0; @@ -947,6 +947,12 @@ format_float_internal(PyObject *value, from a hard-code pseudo-locale */ LocaleInfo locale; + if (format->precision > INT_MAX) { + PyErr_SetString(PyExc_ValueError, "precision too big"); + goto done; + } + precision = (int)format->precision; + /* Alternate is not allowed on floats. */ if (format->alternate) { PyErr_SetString(PyExc_ValueError, @@ -1078,7 +1084,7 @@ format_complex_internal(PyObject *value, Py_ssize_t n_im_total; int re_has_decimal; int im_has_decimal; - Py_ssize_t precision = format->precision; + Py_ssize_t precision; Py_ssize_t default_precision = 6; STRINGLIB_CHAR type = format->type; STRINGLIB_CHAR *p_re; @@ -1107,6 +1113,12 @@ format_complex_internal(PyObject *value, from a hard-code pseudo-locale */ LocaleInfo locale; + if (format->precision > INT_MAX) { + PyErr_SetString(PyExc_ValueError, "precision too big"); + goto done; + } + precision = (int)format->precision; + /* Alternate is not allowed on complex. */ if (format->alternate) { PyErr_SetString(PyExc_ValueError, diff --git a/Objects/stringobject.c b/Objects/stringobject.c index 12091973779..e1ea3cd80f7 100644 --- a/Objects/stringobject.c +++ b/Objects/stringobject.c @@ -882,9 +882,9 @@ string_print(PyStringObject *op, FILE *fp, int flags) size -= chunk_size; } #ifdef __VMS - if (size) fwrite(data, (int)size, 1, fp); + if (size) fwrite(data, (size_t)size, 1, fp); #else - fwrite(data, 1, (int)size, fp); + fwrite(data, 1, (size_t)size, fp); #endif Py_END_ALLOW_THREADS return 0; @@ -1255,7 +1255,6 @@ _PyString_Eq(PyObject *o1, PyObject *o2) PyStringObject *a = (PyStringObject*) o1; PyStringObject *b = (PyStringObject*) o2; return Py_SIZE(a) == Py_SIZE(b) - && *a->ob_sval == *b->ob_sval && memcmp(a->ob_sval, b->ob_sval, Py_SIZE(a)) == 0; } @@ -2332,7 +2331,7 @@ return_self(PyStringObject *self) } Py_LOCAL_INLINE(Py_ssize_t) -countchar(const char *target, int target_len, char c, Py_ssize_t maxcount) +countchar(const char *target, Py_ssize_t target_len, char c, Py_ssize_t maxcount) { Py_ssize_t count=0; const char *start=target; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 518d6e8e052..be04c9e9140 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -225,6 +225,7 @@ static int type_set_name(PyTypeObject *type, PyObject *value, void *context) { PyHeapTypeObject* et; + PyObject *tmp; if (!(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { PyErr_Format(PyExc_TypeError, @@ -253,10 +254,13 @@ type_set_name(PyTypeObject *type, PyObject *value, void *context) Py_INCREF(value); - Py_DECREF(et->ht_name); + /* Wait until et is a sane state before Py_DECREF'ing the old et->ht_name + value. (Bug #16447.) */ + tmp = et->ht_name; et->ht_name = value; type->tp_name = PyString_AS_STRING(value); + Py_DECREF(tmp); return 0; } @@ -5795,15 +5799,16 @@ slot_tp_del(PyObject *self) } -/* Table mapping __foo__ names to tp_foo offsets and slot_tp_foo wrapper - functions. The offsets here are relative to the 'PyHeapTypeObject' - structure, which incorporates the additional structures used for numbers, - sequences and mappings. - Note that multiple names may map to the same slot (e.g. __eq__, - __ne__ etc. all map to tp_richcompare) and one name may map to multiple - slots (e.g. __str__ affects tp_str as well as tp_repr). The table is - terminated with an all-zero entry. (This table is further initialized and - sorted in init_slotdefs() below.) */ +/* +Table mapping __foo__ names to tp_foo offsets and slot_tp_foo wrapper functions. + +The table is ordered by offsets relative to the 'PyHeapTypeObject' structure, +which incorporates the additional structures used for numbers, sequences and +mappings. Note that multiple names may map to the same slot (e.g. __eq__, +__ne__ etc. all map to tp_richcompare) and one name may map to multiple slots +(e.g. __str__ affects tp_str as well as tp_repr). The table is terminated with +an all-zero entry. (This table is further initialized in init_slotdefs().) +*/ typedef struct wrapperbase slotdef; @@ -5853,57 +5858,57 @@ typedef struct wrapperbase slotdef; "x." NAME "(y) <==> " DOC) static slotdef slotdefs[] = { - SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc, - "x.__len__() <==> len(x)"), - /* Heap types defining __add__/__mul__ have sq_concat/sq_repeat == NULL. - The logic in abstract.c always falls back to nb_add/nb_multiply in - this case. Defining both the nb_* and the sq_* slots to call the - user-defined methods has unexpected side-effects, as shown by - test_descr.notimplemented() */ - SQSLOT("__add__", sq_concat, NULL, wrap_binaryfunc, - "x.__add__(y) <==> x+y"), - SQSLOT("__mul__", sq_repeat, NULL, wrap_indexargfunc, - "x.__mul__(n) <==> x*n"), - SQSLOT("__rmul__", sq_repeat, NULL, wrap_indexargfunc, - "x.__rmul__(n) <==> n*x"), - SQSLOT("__getitem__", sq_item, slot_sq_item, wrap_sq_item, - "x.__getitem__(y) <==> x[y]"), - SQSLOT("__getslice__", sq_slice, slot_sq_slice, wrap_ssizessizeargfunc, - "x.__getslice__(i, j) <==> x[i:j]\n\ - \n\ - Use of negative indices is not supported."), - SQSLOT("__setitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_setitem, - "x.__setitem__(i, y) <==> x[i]=y"), - SQSLOT("__delitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_delitem, - "x.__delitem__(y) <==> del x[y]"), - SQSLOT("__setslice__", sq_ass_slice, slot_sq_ass_slice, - wrap_ssizessizeobjargproc, - "x.__setslice__(i, j, y) <==> x[i:j]=y\n\ - \n\ - Use of negative indices is not supported."), - SQSLOT("__delslice__", sq_ass_slice, slot_sq_ass_slice, wrap_delslice, - "x.__delslice__(i, j) <==> del x[i:j]\n\ - \n\ - Use of negative indices is not supported."), - SQSLOT("__contains__", sq_contains, slot_sq_contains, wrap_objobjproc, - "x.__contains__(y) <==> y in x"), - SQSLOT("__iadd__", sq_inplace_concat, NULL, - wrap_binaryfunc, "x.__iadd__(y) <==> x+=y"), - SQSLOT("__imul__", sq_inplace_repeat, NULL, - wrap_indexargfunc, "x.__imul__(y) <==> x*=y"), - - MPSLOT("__len__", mp_length, slot_mp_length, wrap_lenfunc, - "x.__len__() <==> len(x)"), - MPSLOT("__getitem__", mp_subscript, slot_mp_subscript, - wrap_binaryfunc, - "x.__getitem__(y) <==> x[y]"), - MPSLOT("__setitem__", mp_ass_subscript, slot_mp_ass_subscript, - wrap_objobjargproc, - "x.__setitem__(i, y) <==> x[i]=y"), - MPSLOT("__delitem__", mp_ass_subscript, slot_mp_ass_subscript, - wrap_delitem, - "x.__delitem__(y) <==> del x[y]"), - + TPSLOT("__str__", tp_print, NULL, NULL, ""), + TPSLOT("__repr__", tp_print, NULL, NULL, ""), + TPSLOT("__getattribute__", tp_getattr, NULL, NULL, ""), + TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""), + TPSLOT("__setattr__", tp_setattr, NULL, NULL, ""), + TPSLOT("__delattr__", tp_setattr, NULL, NULL, ""), + TPSLOT("__cmp__", tp_compare, _PyObject_SlotCompare, wrap_cmpfunc, + "x.__cmp__(y) <==> cmp(x,y)"), + TPSLOT("__repr__", tp_repr, slot_tp_repr, wrap_unaryfunc, + "x.__repr__() <==> repr(x)"), + TPSLOT("__hash__", tp_hash, slot_tp_hash, wrap_hashfunc, + "x.__hash__() <==> hash(x)"), + FLSLOT("__call__", tp_call, slot_tp_call, (wrapperfunc)wrap_call, + "x.__call__(...) <==> x(...)", PyWrapperFlag_KEYWORDS), + TPSLOT("__str__", tp_str, slot_tp_str, wrap_unaryfunc, + "x.__str__() <==> str(x)"), + TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook, + wrap_binaryfunc, "x.__getattribute__('name') <==> x.name"), + TPSLOT("__getattr__", tp_getattro, slot_tp_getattr_hook, NULL, ""), + TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr, + "x.__setattr__('name', value) <==> x.name = value"), + TPSLOT("__delattr__", tp_setattro, slot_tp_setattro, wrap_delattr, + "x.__delattr__('name') <==> del x.name"), + TPSLOT("__lt__", tp_richcompare, slot_tp_richcompare, richcmp_lt, + "x.__lt__(y) <==> x x<=y"), + TPSLOT("__eq__", tp_richcompare, slot_tp_richcompare, richcmp_eq, + "x.__eq__(y) <==> x==y"), + TPSLOT("__ne__", tp_richcompare, slot_tp_richcompare, richcmp_ne, + "x.__ne__(y) <==> x!=y"), + TPSLOT("__gt__", tp_richcompare, slot_tp_richcompare, richcmp_gt, + "x.__gt__(y) <==> x>y"), + TPSLOT("__ge__", tp_richcompare, slot_tp_richcompare, richcmp_ge, + "x.__ge__(y) <==> x>=y"), + TPSLOT("__iter__", tp_iter, slot_tp_iter, wrap_unaryfunc, + "x.__iter__() <==> iter(x)"), + TPSLOT("next", tp_iternext, slot_tp_iternext, wrap_next, + "x.next() -> the next value, or raise StopIteration"), + TPSLOT("__get__", tp_descr_get, slot_tp_descr_get, wrap_descr_get, + "descr.__get__(obj[, type]) -> value"), + TPSLOT("__set__", tp_descr_set, slot_tp_descr_set, wrap_descr_set, + "descr.__set__(obj, value)"), + TPSLOT("__delete__", tp_descr_set, slot_tp_descr_set, + wrap_descr_delete, "descr.__delete__(obj)"), + FLSLOT("__init__", tp_init, slot_tp_init, (wrapperfunc)wrap_init, + "x.__init__(...) initializes x; " + "see help(type(x)) for signature", + PyWrapperFlag_KEYWORDS), + TPSLOT("__new__", tp_new, slot_tp_new, NULL, ""), + TPSLOT("__del__", tp_del, slot_tp_del, NULL, ""), BINSLOT("__add__", nb_add, slot_nb_add, "+"), RBINSLOT("__radd__", nb_add, slot_nb_add, @@ -5961,8 +5966,6 @@ static slotdef slotdefs[] = { "oct(x)"), UNSLOT("__hex__", nb_hex, slot_nb_hex, wrap_unaryfunc, "hex(x)"), - NBSLOT("__index__", nb_index, slot_nb_index, wrap_unaryfunc, - "x[y:z] <==> x[y.__index__():z.__index__()]"), IBSLOT("__iadd__", nb_inplace_add, slot_nb_inplace_add, wrap_binaryfunc, "+="), IBSLOT("__isub__", nb_inplace_subtract, slot_nb_inplace_subtract, @@ -5993,58 +5996,57 @@ static slotdef slotdefs[] = { slot_nb_inplace_floor_divide, wrap_binaryfunc, "//"), IBSLOT("__itruediv__", nb_inplace_true_divide, slot_nb_inplace_true_divide, wrap_binaryfunc, "/"), - - TPSLOT("__str__", tp_str, slot_tp_str, wrap_unaryfunc, - "x.__str__() <==> str(x)"), - TPSLOT("__str__", tp_print, NULL, NULL, ""), - TPSLOT("__repr__", tp_repr, slot_tp_repr, wrap_unaryfunc, - "x.__repr__() <==> repr(x)"), - TPSLOT("__repr__", tp_print, NULL, NULL, ""), - TPSLOT("__cmp__", tp_compare, _PyObject_SlotCompare, wrap_cmpfunc, - "x.__cmp__(y) <==> cmp(x,y)"), - TPSLOT("__hash__", tp_hash, slot_tp_hash, wrap_hashfunc, - "x.__hash__() <==> hash(x)"), - FLSLOT("__call__", tp_call, slot_tp_call, (wrapperfunc)wrap_call, - "x.__call__(...) <==> x(...)", PyWrapperFlag_KEYWORDS), - TPSLOT("__getattribute__", tp_getattro, slot_tp_getattr_hook, - wrap_binaryfunc, "x.__getattribute__('name') <==> x.name"), - TPSLOT("__getattribute__", tp_getattr, NULL, NULL, ""), - TPSLOT("__getattr__", tp_getattro, slot_tp_getattr_hook, NULL, ""), - TPSLOT("__getattr__", tp_getattr, NULL, NULL, ""), - TPSLOT("__setattr__", tp_setattro, slot_tp_setattro, wrap_setattr, - "x.__setattr__('name', value) <==> x.name = value"), - TPSLOT("__setattr__", tp_setattr, NULL, NULL, ""), - TPSLOT("__delattr__", tp_setattro, slot_tp_setattro, wrap_delattr, - "x.__delattr__('name') <==> del x.name"), - TPSLOT("__delattr__", tp_setattr, NULL, NULL, ""), - TPSLOT("__lt__", tp_richcompare, slot_tp_richcompare, richcmp_lt, - "x.__lt__(y) <==> x x<=y"), - TPSLOT("__eq__", tp_richcompare, slot_tp_richcompare, richcmp_eq, - "x.__eq__(y) <==> x==y"), - TPSLOT("__ne__", tp_richcompare, slot_tp_richcompare, richcmp_ne, - "x.__ne__(y) <==> x!=y"), - TPSLOT("__gt__", tp_richcompare, slot_tp_richcompare, richcmp_gt, - "x.__gt__(y) <==> x>y"), - TPSLOT("__ge__", tp_richcompare, slot_tp_richcompare, richcmp_ge, - "x.__ge__(y) <==> x>=y"), - TPSLOT("__iter__", tp_iter, slot_tp_iter, wrap_unaryfunc, - "x.__iter__() <==> iter(x)"), - TPSLOT("next", tp_iternext, slot_tp_iternext, wrap_next, - "x.next() -> the next value, or raise StopIteration"), - TPSLOT("__get__", tp_descr_get, slot_tp_descr_get, wrap_descr_get, - "descr.__get__(obj[, type]) -> value"), - TPSLOT("__set__", tp_descr_set, slot_tp_descr_set, wrap_descr_set, - "descr.__set__(obj, value)"), - TPSLOT("__delete__", tp_descr_set, slot_tp_descr_set, - wrap_descr_delete, "descr.__delete__(obj)"), - FLSLOT("__init__", tp_init, slot_tp_init, (wrapperfunc)wrap_init, - "x.__init__(...) initializes x; " - "see help(type(x)) for signature", - PyWrapperFlag_KEYWORDS), - TPSLOT("__new__", tp_new, slot_tp_new, NULL, ""), - TPSLOT("__del__", tp_del, slot_tp_del, NULL, ""), + NBSLOT("__index__", nb_index, slot_nb_index, wrap_unaryfunc, + "x[y:z] <==> x[y.__index__():z.__index__()]"), + MPSLOT("__len__", mp_length, slot_mp_length, wrap_lenfunc, + "x.__len__() <==> len(x)"), + MPSLOT("__getitem__", mp_subscript, slot_mp_subscript, + wrap_binaryfunc, + "x.__getitem__(y) <==> x[y]"), + MPSLOT("__setitem__", mp_ass_subscript, slot_mp_ass_subscript, + wrap_objobjargproc, + "x.__setitem__(i, y) <==> x[i]=y"), + MPSLOT("__delitem__", mp_ass_subscript, slot_mp_ass_subscript, + wrap_delitem, + "x.__delitem__(y) <==> del x[y]"), + SQSLOT("__len__", sq_length, slot_sq_length, wrap_lenfunc, + "x.__len__() <==> len(x)"), + /* Heap types defining __add__/__mul__ have sq_concat/sq_repeat == NULL. + The logic in abstract.c always falls back to nb_add/nb_multiply in + this case. Defining both the nb_* and the sq_* slots to call the + user-defined methods has unexpected side-effects, as shown by + test_descr.notimplemented() */ + SQSLOT("__add__", sq_concat, NULL, wrap_binaryfunc, + "x.__add__(y) <==> x+y"), + SQSLOT("__mul__", sq_repeat, NULL, wrap_indexargfunc, + "x.__mul__(n) <==> x*n"), + SQSLOT("__rmul__", sq_repeat, NULL, wrap_indexargfunc, + "x.__rmul__(n) <==> n*x"), + SQSLOT("__getitem__", sq_item, slot_sq_item, wrap_sq_item, + "x.__getitem__(y) <==> x[y]"), + SQSLOT("__getslice__", sq_slice, slot_sq_slice, wrap_ssizessizeargfunc, + "x.__getslice__(i, j) <==> x[i:j]\n\ + \n\ + Use of negative indices is not supported."), + SQSLOT("__setitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_setitem, + "x.__setitem__(i, y) <==> x[i]=y"), + SQSLOT("__delitem__", sq_ass_item, slot_sq_ass_item, wrap_sq_delitem, + "x.__delitem__(y) <==> del x[y]"), + SQSLOT("__setslice__", sq_ass_slice, slot_sq_ass_slice, + wrap_ssizessizeobjargproc, + "x.__setslice__(i, j, y) <==> x[i:j]=y\n\ + \n\ + Use of negative indices is not supported."), + SQSLOT("__delslice__", sq_ass_slice, slot_sq_ass_slice, wrap_delslice, + "x.__delslice__(i, j) <==> del x[i:j]\n\ + \n\ + Use of negative indices is not supported."), + SQSLOT("__contains__", sq_contains, slot_sq_contains, wrap_objobjproc, + "x.__contains__(y) <==> y in x"), + SQSLOT("__iadd__", sq_inplace_concat, NULL, + wrap_binaryfunc, "x.__iadd__(y) <==> x+=y"), + SQSLOT("__imul__", sq_inplace_repeat, NULL, + wrap_indexargfunc, "x.__imul__(y) <==> x*=y"), {NULL} }; @@ -6225,21 +6227,6 @@ update_slots_callback(PyTypeObject *type, void *data) return 0; } -/* Comparison function for qsort() to compare slotdefs by their offset, and - for equal offset by their address (to force a stable sort). */ -static int -slotdef_cmp(const void *aa, const void *bb) -{ - const slotdef *a = (const slotdef *)aa, *b = (const slotdef *)bb; - int c = a->offset - b->offset; - if (c != 0) - return c; - else - /* Cannot use a-b, as this gives off_t, - which may lose precision when converted to int. */ - return (a > b) ? 1 : (a < b) ? -1 : 0; -} - /* Initialize the slotdefs table by adding interned string objects for the names and sorting the entries. */ static void @@ -6251,12 +6238,12 @@ init_slotdefs(void) if (initialized) return; for (p = slotdefs; p->name; p++) { + /* Slots must be ordered by their offset in the PyHeapTypeObject. */ + assert(!p[1].name || p->offset <= p[1].offset); p->name_strobj = PyString_InternFromString(p->name); if (!p->name_strobj) Py_FatalError("Out of memory interning slotdef names"); } - qsort((void *)slotdefs, (size_t)(p-slotdefs), sizeof(slotdef), - slotdef_cmp); initialized = 1; } diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 0ead06f242c..866eb9b0589 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -547,6 +547,37 @@ PyObject *PyUnicode_FromString(const char *u) return PyUnicode_FromStringAndSize(u, size); } +/* _Py_UNICODE_NEXT is a private macro used to retrieve the character pointed + * by 'ptr', possibly combining surrogate pairs on narrow builds. + * 'ptr' and 'end' must be Py_UNICODE*, with 'ptr' pointing at the character + * that should be returned and 'end' pointing to the end of the buffer. + * ('end' is used on narrow builds to detect a lone surrogate at the + * end of the buffer that should be returned unchanged.) + * The ptr and end arguments should be side-effect free and ptr must an lvalue. + * The type of the returned char is always Py_UCS4. + * + * Note: the macro advances ptr to next char, so it might have side-effects + * (especially if used with other macros). + */ + +/* helper macros used by _Py_UNICODE_NEXT */ +#define _Py_UNICODE_IS_HIGH_SURROGATE(ch) (0xD800 <= ch && ch <= 0xDBFF) +#define _Py_UNICODE_IS_LOW_SURROGATE(ch) (0xDC00 <= ch && ch <= 0xDFFF) +/* Join two surrogate characters and return a single Py_UCS4 value. */ +#define _Py_UNICODE_JOIN_SURROGATES(high, low) \ + (((((Py_UCS4)(high) & 0x03FF) << 10) | \ + ((Py_UCS4)(low) & 0x03FF)) + 0x10000) + +#ifdef Py_UNICODE_WIDE +#define _Py_UNICODE_NEXT(ptr, end) *(ptr)++ +#else +#define _Py_UNICODE_NEXT(ptr, end) \ + (((_Py_UNICODE_IS_HIGH_SURROGATE(*(ptr)) && (ptr) < (end)) && \ + _Py_UNICODE_IS_LOW_SURROGATE((ptr)[1])) ? \ + ((ptr) += 2,_Py_UNICODE_JOIN_SURROGATES((ptr)[-2], (ptr)[-1])) : \ + (Py_UCS4)*(ptr)++) +#endif + #ifdef HAVE_WCHAR_H #if (Py_UNICODE_SIZE == 2) && defined(SIZEOF_WCHAR_T) && (SIZEOF_WCHAR_T == 4) @@ -740,8 +771,25 @@ PyUnicode_FromFormatV(const char *format, va_list vargs) switch (*f) { case 'c': - (void)va_arg(count, int); + { + int ordinal = va_arg(count, int); +#ifdef Py_UNICODE_WIDE + if (ordinal < 0 || ordinal > 0x10ffff) { + PyErr_SetString(PyExc_OverflowError, + "%c arg not in range(0x110000) " + "(wide Python build)"); + goto fail; + } +#else + if (ordinal < 0 || ordinal > 0xffff) { + PyErr_SetString(PyExc_OverflowError, + "%c arg not in range(0x10000) " + "(narrow Python build)"); + goto fail; + } +#endif /* fall through... */ + } case '%': n++; break; @@ -3625,26 +3673,22 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p, case 4: /* xmlcharrefreplace */ respos = str-PyString_AS_STRING(res); /* determine replacement size (temporarily (mis)uses p) */ - for (p = collstart, repsize = 0; p < collend; ++p) { - if (*p<10) + for (p = collstart, repsize = 0; p < collend;) { + Py_UCS4 ch = _Py_UNICODE_NEXT(p, collend); + if (ch < 10) repsize += 2+1+1; - else if (*p<100) + else if (ch < 100) repsize += 2+2+1; - else if (*p<1000) + else if (ch < 1000) repsize += 2+3+1; - else if (*p<10000) + else if (ch < 10000) repsize += 2+4+1; -#ifndef Py_UNICODE_WIDE - else - repsize += 2+5+1; -#else - else if (*p<100000) + else if (ch < 100000) repsize += 2+5+1; - else if (*p<1000000) + else if (ch < 1000000) repsize += 2+6+1; else repsize += 2+7+1; -#endif } requiredsize = respos+repsize+(endp-collend); if (requiredsize > ressize) { @@ -3656,8 +3700,9 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p, ressize = requiredsize; } /* generate replacement (temporarily (mis)uses p) */ - for (p = collstart; p < collend; ++p) { - str += sprintf(str, "&#%d;", (int)*p); + for (p = collstart; p < collend;) { + Py_UCS4 ch = _Py_UNICODE_NEXT(p, collend); + str += sprintf(str, "&#%d;", (int)ch); } p = collend; break; @@ -4632,11 +4677,20 @@ int charmap_encoding_error( *inpos = collendpos; break; case 4: /* xmlcharrefreplace */ - /* generate replacement (temporarily (mis)uses p) */ - for (collpos = collstartpos; collpos < collendpos; ++collpos) { + /* generate replacement */ + for (collpos = collstartpos; collpos < collendpos;) { char buffer[2+29+1+1]; char *cp; - sprintf(buffer, "&#%d;", (int)p[collpos]); + Py_UCS4 ch = p[collpos++]; +#ifndef Py_UNICODE_WIDE + if ((0xD800 <= ch && ch <= 0xDBFF) && + (collpos < collendpos) && + (0xDC00 <= p[collpos] && p[collpos] <= 0xDFFF)) { + ch = ((((ch & 0x03FF) << 10) | + ((Py_UCS4)p[collpos++] & 0x03FF)) + 0x10000); + } +#endif + sprintf(buffer, "&#%d;", (int)ch); for (cp = buffer; *cp; ++cp) { x = charmapencode_output(*cp, mapping, res, respos); if (x==enc_EXCEPTION) @@ -5051,10 +5105,11 @@ PyObject *PyUnicode_TranslateCharmap(const Py_UNICODE *p, break; case 4: /* xmlcharrefreplace */ /* generate replacement (temporarily (mis)uses p) */ - for (p = collstart; p < collend; ++p) { + for (p = collstart; p < collend;) { char buffer[2+29+1+1]; char *cp; - sprintf(buffer, "&#%d;", (int)*p); + Py_UCS4 ch = _Py_UNICODE_NEXT(p, collend); + sprintf(buffer, "&#%d;", (int)ch); if (charmaptranslate_makespace(&res, &str, (str-PyUnicode_AS_UNICODE(res))+strlen(buffer)+(endp-collend))) goto onError; @@ -5205,8 +5260,10 @@ int PyUnicode_EncodeDecimal(Py_UNICODE *s, break; case 4: /* xmlcharrefreplace */ /* generate replacement (temporarily (mis)uses p) */ - for (p = collstart; p < collend; ++p) - output += sprintf(output, "&#%d;", (int)*p); + for (p = collstart; p < collend;) { + Py_UCS4 ch = _Py_UNICODE_NEXT(p, collend); + output += sprintf(output, "&#%d;", ch); + } p = collend; break; default: diff --git a/PC/_subprocess.c b/PC/_subprocess.c index 689b0c8b433..195e343decb 100644 --- a/PC/_subprocess.c +++ b/PC/_subprocess.c @@ -331,7 +331,7 @@ getenvironment(PyObject* environment) PyObject* values; char* p; - /* convert environment dictionary to windows enviroment string */ + /* convert environment dictionary to windows environment string */ if (! PyMapping_Check(environment)) { PyErr_SetString( PyExc_TypeError, "environment must be dictionary or None"); diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 3395aac1bbd..00b50ece97d 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -262,7 +262,7 @@ SDK, in particular the 64-bit support. This includes an Itanium compiler In addition, you need the Visual Studio plugin for external C compilers, from http://sf.net/projects/vsextcomp. The plugin will wrap cl.exe, to locate the proper target compiler, and convert compiler options -accordingly. The project files require atleast version 0.9. +accordingly. The project files require at least version 0.9. Building for AMD64 ------------------ diff --git a/PCbuild/rt.bat b/PCbuild/rt.bat index ee1661a21ab..9da1ed1b397 100644 --- a/PCbuild/rt.bat +++ b/PCbuild/rt.bat @@ -30,7 +30,7 @@ set prefix=.\ set suffix= set qmode= set dashO= -set tcltk= +set tcltk=tcltk :CheckOpts if "%1"=="-O" (set dashO=-O) & shift & goto CheckOpts @@ -38,7 +38,7 @@ if "%1"=="-q" (set qmode=yes) & shift & goto CheckOpts if "%1"=="-d" (set suffix=_d) & shift & goto CheckOpts if "%1"=="-x64" (set prefix=amd64) & (set tcltk=tcltk64) & shift & goto CheckOpts -PATH %PATH%;..\..\%tcltk%\bin +PATH %PATH%;%~dp0..\..\%tcltk%\bin set exe=%prefix%\python%suffix% set cmd=%exe% %dashO% -Wd -3 -E -tt ../lib/test/regrtest.py %1 %2 %3 %4 %5 %6 %7 %8 %9 if defined qmode goto Qmode diff --git a/README b/PYTHON-README similarity index 99% rename from README rename to PYTHON-README index 9b3f16eea49..2b9fc69252f 100644 --- a/README +++ b/PYTHON-README @@ -1,4 +1,4 @@ -This is Python version 2.7.4 +This is Python version 2.7.5 ============================ Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, @@ -89,6 +89,13 @@ reStructuredText (2.6+) formats; the LaTeX and reStructuredText versions are primarily for documentation authors, translators, and people with special formatting requirements. +If you would like to contribute to the development of Python, relevant +documentation is available at: + + http://docs.python.org/devguide/ + +For information about building Python's documentation, refer to Doc/README.txt. + Web sites --------- diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 666c48d199b..7ebc2369e38 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -977,7 +977,7 @@ def has_sequence(types, doing_specialization): class StaticVisitor(PickleVisitor): - CODE = '''Very simple, always emit this static code. Overide CODE''' + CODE = '''Very simple, always emit this static code. Override CODE''' def visit(self, object): self.emit(self.CODE, 0, reflow=False) diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index ee6313b311d..46cf9b297cd 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -277,8 +277,11 @@ check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, tok->encoding = cs; tok->decoding_state = -1; } - else + else { + PyErr_Format(PyExc_SyntaxError, + "encoding problem: %s", cs); PyMem_FREE(cs); + } #else /* Without Unicode support, we cannot process the coding spec. Since there @@ -289,15 +292,12 @@ check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, } } else { /* then, compare cs with BOM */ r = (strcmp(tok->encoding, cs) == 0); + if (!r) + PyErr_Format(PyExc_SyntaxError, + "encoding problem: %s with BOM", cs); PyMem_FREE(cs); } } - if (!r) { - cs = tok->encoding; - if (!cs) - cs = "with BOM"; - PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs); - } return r; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index cdc908084fc..d22dca2f487 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2434,9 +2434,9 @@ builtin_sum(PyObject *self, PyObject *args) PyDoc_STRVAR(sum_doc, "sum(sequence[, start]) -> value\n\ \n\ -Returns the sum of a sequence of numbers (NOT strings) plus the value\n\ +Return the sum of a sequence of numbers (NOT strings) plus the value\n\ of parameter 'start' (which defaults to 0). When the sequence is\n\ -empty, returns start."); +empty, return start."); static PyObject * diff --git a/Python/codecs.c b/Python/codecs.c index 7334eb3e360..69498c4b0c9 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -521,7 +521,7 @@ PyObject *PyCodec_ReplaceErrors(PyObject *exc) Py_UNICODE res = Py_UNICODE_REPLACEMENT_CHARACTER; if (PyUnicodeDecodeError_GetEnd(exc, &end)) return NULL; - return Py_BuildValue("(u#n)", &res, 1, end); + return Py_BuildValue("(u#n)", &res, (Py_ssize_t)1, end); } else if (PyObject_IsInstance(exc, PyExc_UnicodeTranslateError)) { PyObject *res; @@ -556,6 +556,7 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc) PyObject *res; Py_UNICODE *p; Py_UNICODE *startp; + Py_UNICODE *e; Py_UNICODE *outp; int ressize; if (PyUnicodeEncodeError_GetStart(exc, &start)) @@ -565,26 +566,31 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc) if (!(object = PyUnicodeEncodeError_GetObject(exc))) return NULL; startp = PyUnicode_AS_UNICODE(object); - for (p = startp+start, ressize = 0; p < startp+end; ++p) { - if (*p<10) + e = startp + end; + for (p = startp+start, ressize = 0; p < e;) { + Py_UCS4 ch = *p++; +#ifndef Py_UNICODE_WIDE + if ((0xD800 <= ch && ch <= 0xDBFF) && + (p < e) && + (0xDC00 <= *p && *p <= 0xDFFF)) { + ch = ((((ch & 0x03FF) << 10) | + ((Py_UCS4)*p++ & 0x03FF)) + 0x10000); + } +#endif + if (ch < 10) ressize += 2+1+1; - else if (*p<100) + else if (ch < 100) ressize += 2+2+1; - else if (*p<1000) + else if (ch < 1000) ressize += 2+3+1; - else if (*p<10000) + else if (ch < 10000) ressize += 2+4+1; -#ifndef Py_UNICODE_WIDE - else - ressize += 2+5+1; -#else - else if (*p<100000) + else if (ch < 100000) ressize += 2+5+1; - else if (*p<1000000) + else if (ch < 1000000) ressize += 2+6+1; else ressize += 2+7+1; -#endif } /* allocate replacement */ res = PyUnicode_FromUnicode(NULL, ressize); @@ -593,40 +599,41 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc) return NULL; } /* generate replacement */ - for (p = startp+start, outp = PyUnicode_AS_UNICODE(res); - p < startp+end; ++p) { - Py_UNICODE c = *p; + for (p = startp+start, outp = PyUnicode_AS_UNICODE(res); p < e;) { int digits; int base; + Py_UCS4 ch = *p++; +#ifndef Py_UNICODE_WIDE + if ((0xD800 <= ch && ch <= 0xDBFF) && + (p < startp+end) && + (0xDC00 <= *p && *p <= 0xDFFF)) { + ch = ((((ch & 0x03FF) << 10) | + ((Py_UCS4)*p++ & 0x03FF)) + 0x10000); + } +#endif *outp++ = '&'; *outp++ = '#'; - if (*p<10) { + if (ch < 10) { digits = 1; base = 1; } - else if (*p<100) { + else if (ch < 100) { digits = 2; base = 10; } - else if (*p<1000) { + else if (ch < 1000) { digits = 3; base = 100; } - else if (*p<10000) { + else if (ch < 10000) { digits = 4; base = 1000; } -#ifndef Py_UNICODE_WIDE - else { - digits = 5; - base = 10000; - } -#else - else if (*p<100000) { + else if (ch < 100000) { digits = 5; base = 10000; } - else if (*p<1000000) { + else if (ch < 1000000) { digits = 6; base = 100000; } @@ -634,10 +641,9 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc) digits = 7; base = 1000000; } -#endif while (digits-->0) { - *outp++ = '0' + c/base; - c %= base; + *outp++ = '0' + ch/base; + ch %= base; base /= 10; } *outp++ = ';'; diff --git a/Python/compile.c b/Python/compile.c index 531bed43c48..8354c75b393 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -221,8 +221,11 @@ _Py_Mangle(PyObject *privateobj, PyObject *ident) } plen = strlen(p); - assert(1 <= PY_SSIZE_T_MAX - nlen); - assert(1 + nlen <= PY_SSIZE_T_MAX - plen); + if (plen + nlen >= PY_SSIZE_T_MAX - 1) { + PyErr_SetString(PyExc_OverflowError, + "private identifier too large to be mangled"); + return NULL; + } ident = PyString_FromStringAndSize(NULL, 1 + nlen + plen); if (!ident) diff --git a/Python/marshal.c b/Python/marshal.c index 33685b9d7c2..6b285aaa41e 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -88,7 +88,7 @@ w_more(int c, WFILE *p) } static void -w_string(char *s, Py_ssize_t n, WFILE *p) +w_string(const char *s, Py_ssize_t n, WFILE *p) { if (p->fp != NULL) { fwrite(s, 1, n, p->fp); @@ -141,6 +141,13 @@ w_long64(long x, WFILE *p) # define W_SIZE w_long #endif +static void +w_pstring(const char *s, Py_ssize_t n, WFILE *p) +{ + W_SIZE(n, p); + w_string(s, n, p); +} + /* We assume that Python longs are stored internally in base some power of 2**15; for the sake of portability we'll always read and write them in base exactly 2**15. */ @@ -338,9 +345,7 @@ w_object(PyObject *v, WFILE *p) else { w_byte(TYPE_STRING, p); } - n = PyString_GET_SIZE(v); - W_SIZE(n, p); - w_string(PyString_AS_STRING(v), n, p); + w_pstring(PyBytes_AS_STRING(v), PyString_GET_SIZE(v), p); } #ifdef Py_USING_UNICODE else if (PyUnicode_CheckExact(v)) { @@ -352,9 +357,7 @@ w_object(PyObject *v, WFILE *p) return; } w_byte(TYPE_UNICODE, p); - n = PyString_GET_SIZE(utf8); - W_SIZE(n, p); - w_string(PyString_AS_STRING(utf8), n, p); + w_pstring(PyString_AS_STRING(utf8), PyString_GET_SIZE(utf8), p); Py_DECREF(utf8); } #endif @@ -441,8 +444,7 @@ w_object(PyObject *v, WFILE *p) PyBufferProcs *pb = v->ob_type->tp_as_buffer; w_byte(TYPE_STRING, p); n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s); - W_SIZE(n, p); - w_string(s, n, p); + w_pstring(s, n, p); } else { w_byte(TYPE_UNKNOWN, p); diff --git a/Python/random.c b/Python/random.c index 825260f36b2..d615923dd2f 100644 --- a/Python/random.c +++ b/Python/random.c @@ -165,8 +165,12 @@ dev_urandom_python(char *buffer, Py_ssize_t size) Py_END_ALLOW_THREADS if (fd < 0) { - PyErr_SetString(PyExc_NotImplementedError, - "/dev/urandom (or equivalent) not found"); + if (errno == ENOENT || errno == ENXIO || + errno == ENODEV || errno == EACCES) + PyErr_SetString(PyExc_NotImplementedError, + "/dev/urandom (or equivalent) not found"); + else + PyErr_SetFromErrno(PyExc_OSError); return -1; } @@ -220,8 +224,9 @@ lcg_urandom(unsigned int x0, unsigned char *buffer, size_t size) } } -/* Fill buffer with size pseudo-random bytes, not suitable for cryptographic - use, from the operating random number generator (RNG). +/* Fill buffer with size pseudo-random bytes from the operating system random + number generator (RNG). It is suitable for for most cryptographic purposes + except long living private keys for asymmetric encryption. Return 0 on success, raise an exception and return -1 on error. */ int diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 814eccb1571..a868715ffcf 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -219,7 +219,7 @@ PyDoc_STRVAR(exit_doc, \n\ Exit the interpreter by raising SystemExit(status).\n\ If the status is omitted or None, it defaults to zero (i.e., success).\n\ -If the status is numeric, it will be used as the system exit status.\n\ +If the status is an integer, it will be used as the system exit status.\n\ If it is another kind of object, it will be printed and the system\n\ exit status will be one (i.e., failure)." ); diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index c1c92d1a15e..c9ed796cd0f 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -145,6 +145,7 @@ static void PyThread__init_thread(void) { #if defined(_AIX) && defined(__GNUC__) + extern void pthread_init(void); pthread_init(); #endif } @@ -394,6 +395,7 @@ PyThread_free_lock(PyThread_type_lock lock) pthread_lock *thelock = (pthread_lock *)lock; int status, error = 0; + (void) error; /* silence unused-but-set-variable warning */ dprintf(("PyThread_free_lock(%p) called\n", lock)); status = pthread_mutex_destroy( &thelock->mut ); @@ -445,6 +447,7 @@ PyThread_release_lock(PyThread_type_lock lock) pthread_lock *thelock = (pthread_lock *)lock; int status, error = 0; + (void) error; /* silence unused-but-set-variable warning */ dprintf(("PyThread_release_lock(%p) called\n", lock)); status = pthread_mutex_lock( &thelock->mut ); diff --git a/README.md b/README.md new file mode 100644 index 00000000000..62502b29e0f --- /dev/null +++ b/README.md @@ -0,0 +1,99 @@ +Static-Python +============= + +This is a fork of the official Python hg repository with additional tools to +enable building Python for static linking. + +You may be wondering, "why would you want to do that? After all, [static linking +is evil](http://www.akkadia.org/drepper/no_static_linking.html)." Here are a few +possible reasons: + + * To run Python programs on other machines without requiring that they install + Python. + + * To run Python programs on other machines without requiring that they have + the same versions of the same libraries installed that you do. + + * Because the major binary distribution tools for Python (cx_Freeze, bbfreeze, + py2exe, and py2app) ship in a way that the Python source code can be + trivially derived (unzip the archive of .pyc files and decompile them.) For + proprietary or security-conscious applications, this is unacceptable. + + +Usage +===== + +Building Static Python +---------------------- + +To build a static Python executable and library, check out the appropriate branch +(either 2.7, 3.3, or master) and run the following command: + + make -f Static.make + +This will create an executable called `python` in the working directory, and a +static library, `libpythonX.X.a`, in the install/lib directory. You can confirm +that this executable is not dependent on any shared libraries using `ldd python` +which should report that python is not a dynamic executable. However, by default +this executable's functionality will be very limited - it won't even be able to +access most modules from the Python standard library. + +In order to make this Python interpreter truly standalone (not dependent on +installed Python modules), you can designate Python modules to be compiled as +builtins, which will be statically linked into the Python interpreter. +Static.make generates a file in Modules/Setup which needs to be edited to +specify these new builtin modules. + +You can automatically add builtins when building Static Python by passing +BUILTINS and/or SCRIPT variables to Static.make, e.g.: + + make -f Static.make BUILTINS="math zipfile zlib" SCRIPT="/path/to/script.py" + +Each module listed in the BUILTINS variable will be added, if possible. SCRIPT +can be used to specify the path to a Python script. This script will be scanned +for dependencies using modulefinder, and all dependencies will be added as +builtins if possible (not the script itself - it should be compiled using +static_freeze.py and linked to the resulting static library.) Finally, if the +DFLAG variable is set to "-d", all dependencies of all modules will be +automatically added as well (this will usually include many modules, and you may +not really need them all.) + +(If you previously built Static Python, you should `make -f Static.make clean` +first. Also, this step requires an existing Python installation, preferably of +the same version you're building, so you may need to build and install Python +the normal way first before building it statically..) + +Adding builtins can also be done manually by editing Modules/Setup. Add lines to +the end of the file in the format: + + module_name module.c ... + +These .c files can be generated from .py files using Cython: + + cython module.py + +(This is done automatically using add_builtins.py, the script called by +Static.make when BUILTINS or SCRIPT are supplied.) + +Packages are not currently supported. I'm working on an automatic solution for +compiling and including packages. Stay tuned. + + +Compile a standalone executable +------------------------------- + +Once you've compiled a static Python library, you can turn a Python script into +a standalone executable using the static_freeze.py script. + + Tools/static_freeze/static_freeze.py test.py libpython2.7.a + +This will generate an executable called "test" in the working directory, which +is not dependant on any shared libraries, Python modules, or the Python +interpreter! + + +Acknowledgements +================ + +Major thanks to Gabriel Jacobo who pioneered this method: + diff --git a/Static.make b/Static.make new file mode 100644 index 00000000000..3dd23aab598 --- /dev/null +++ b/Static.make @@ -0,0 +1,35 @@ +.PHONY: all setup clean + +all: python +setup: Modules/Setup + +PYTHON=python +PREFIX=$(shell pwd)/install +CONF_ARGS= +MAKE_ARGS= +BUILTINS= +override BUILTINS+= array cmath math _struct time _operator _random _collections _heapq itertools _functools _elementtree _pickle _datetime _bisect unicodedata atexit _weakref datetime +SCRIPT= +DFLAG= +CPPFLAGS= +LDFLAGS= +INCLUDE=-I/usr/include + +Modules/Setup: Modules/Setup.dist add_builtins.py + sed -e 's/#\*shared\*/\*static\*/g' Modules/Setup.dist \ + > Modules/Setup + [ -d Modules/extras ] || mkdir Modules/extras + $(PYTHON) add_builtins.py $(BUILTINS) $(DFLAG) -s $(SCRIPT) + +Makefile: Modules/Setup + [ -d $(PREFIX) ] || mkdir $(PREFIX) + ./configure LDFLAGS="-Wl,-no-export-dynamic -static-libgcc -static $(LDFLAGS) $(INCLUDE)" \ + CPPFLAGS="-I/usr/lib -static -fPIC $(CPPLAGS) $(INCLUDE)" LINKFORSHARED=" " \ + DYNLOADFILE="dynload_stub.o" -disable-shared \ + -prefix="$(PREFIX)" $(CONF_ARGS) + +python: Modules/Setup Makefile + make $(MAKE_ARGS) + +clean: + rm -f Makefile Modules/Setup \ No newline at end of file diff --git a/Tools/freeze/checkextensions_win32.py b/Tools/freeze/checkextensions_win32.py index e5a8b29bb95..8c6444cf5ad 100644 --- a/Tools/freeze/checkextensions_win32.py +++ b/Tools/freeze/checkextensions_win32.py @@ -3,7 +3,7 @@ Under Windows it is unlikely the .obj files are of use, as special compiler options are needed (primarily to toggle the behavior of "public" symbols. -I dont consider it worth parsing the MSVC makefiles for compiler options. Even if +I don't consider it worth parsing the MSVC makefiles for compiler options. Even if we get it just right, a specific freeze application may have specific compiler options anyway (eg, to enable or disable specific functionality) @@ -14,7 +14,7 @@ your own). * This description can include: - The MSVC .dsp file for the extension. The .c source file names - are extraced from there. + are extracted from there. - Specific compiler/linker options - Flag to indicate if Unicode compilation is expected. diff --git a/Tools/freeze/makefreeze.py b/Tools/freeze/makefreeze.py index 1208b67fe07..c0f50566b0d 100644 --- a/Tools/freeze/makefreeze.py +++ b/Tools/freeze/makefreeze.py @@ -62,7 +62,7 @@ def makefreeze(base, dict, debug=0, entry_point=None, fail_import=()): outfp.write('\t{"%s", M_%s, %d},\n' % (mod, mangled, size)) outfp.write('\n') # The following modules have a NULL code pointer, indicating - # that the prozen program should not search for them on the host + # that the frozen program should not search for them on the host # system. Importing them will *always* raise an ImportError. # The zero value size is never used. for mod in fail_import: diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index e12769d7d27..b4bfdccc32c 100644 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -617,7 +617,7 @@ class PyDictObjectPtr(PyObjectPtr): def iteritems(self): ''' Yields a sequence of (PyObjectPtr key, PyObjectPtr value) pairs, - analagous to dict.iteritems() + analogous to dict.iteritems() ''' for i in safe_range(self.field('ma_mask') + 1): ep = self.field('ma_table') + i diff --git a/Tools/i18n/msgfmt.py b/Tools/i18n/msgfmt.py index 8c9b9dc9a34..2502a109c04 100755 --- a/Tools/i18n/msgfmt.py +++ b/Tools/i18n/msgfmt.py @@ -144,7 +144,7 @@ def make(filename, outfile): # This is a message with plural forms elif l.startswith('msgid_plural'): if section != ID: - print >> sys.stderr, 'msgid_plural not preceeded by msgid on %s:%d' %\ + print >> sys.stderr, 'msgid_plural not preceded by msgid on %s:%d' %\ (infile, lno) sys.exit(1) l = l[12:] diff --git a/Tools/msi/msi.py b/Tools/msi/msi.py index d2caf345b1d..2f0d9637140 100644 --- a/Tools/msi/msi.py +++ b/Tools/msi/msi.py @@ -1392,7 +1392,10 @@ def merge(msi, feature, rootdir, modules): # certname (from config.py) should be (a substring of) # the certificate subject, e.g. "Python Software Foundation" if certname: - os.system('signtool sign /n "%s" /t http://timestamp.verisign.com/scripts/timestamp.dll %s' % (certname, msiname)) + os.system('signtool sign /n "%s" ' + '/t http://timestamp.verisign.com/scripts/timestamp.dll ' + '/d "Python %s" ' + '%s' % (certname, full_current_version, msiname)) if pdbzip: build_pdbzip() diff --git a/Tools/msi/msilib.py b/Tools/msi/msilib.py index e02a3e2316f..760471c37b2 100644 --- a/Tools/msi/msilib.py +++ b/Tools/msi/msilib.py @@ -305,7 +305,7 @@ def init_database(name, schema, t.create(db) # Fill the validation table add_data(db, "_Validation", schema._Validation_records) - # Initialize the summary information, allowing atmost 20 properties + # Initialize the summary information, allowing at most 20 properties si = db.GetSummaryInformation(20) si.SetProperty(PID_TITLE, "Installation Database") si.SetProperty(PID_SUBJECT, ProductName) diff --git a/Tools/msi/uuids.py b/Tools/msi/uuids.py index cf5bfd239f6..7c54e513dec 100644 --- a/Tools/msi/uuids.py +++ b/Tools/msi/uuids.py @@ -60,4 +60,7 @@ '2.7.3150':'{C0C31BCC-56FB-42a7-8766-D29E1BD74C7C}', # 2.7.3 '2.7.4121':'{47F45F45-72D7-4e54-AF41-26767EDE95CF}', # 2.7.4rc1 '2.7.4150':'{84ADC96C-B7E0-4938-9D6E-2B640D5DA224}', # 2.7.4 + '2.7.5150':'{DBDD570E-0952-475f-9453-AB88F3DD5659}', # 2.7.5 + '2.7.6121':'{D1EBC07F-A7B1-4163-83DB-AE813CEF392F}', # 2.7.6rc1 + '2.7.6150':'{C3CC4DF5-39A5-4027-B136-2B3E1F5AB6E2}', # 2.7.6 } diff --git a/Tools/pybench/CommandLine.py b/Tools/pybench/CommandLine.py index 6601be5fb51..fde317841e1 100644 --- a/Tools/pybench/CommandLine.py +++ b/Tools/pybench/CommandLine.py @@ -458,7 +458,7 @@ def parse(self): handler = getattr(self, handlername) except AttributeError: if value == '': - # count the number of occurances + # count the number of occurrences if values.has_key(optionname): values[optionname] = values[optionname] + 1 else: diff --git a/Tools/pybench/systimes.py b/Tools/pybench/systimes.py index 013add278e9..db1210dee22 100644 --- a/Tools/pybench/systimes.py +++ b/Tools/pybench/systimes.py @@ -5,7 +5,7 @@ This module implements various different strategies for measuring performance timings. It tries to choose the best available method - based on the platforma and available tools. + based on the platform and available tools. On Windows, it is recommended to have the Mark Hammond win32 package installed. Alternatively, the Thomas Heller ctypes diff --git a/Tools/pynche/DetailsViewer.py b/Tools/pynche/DetailsViewer.py index 11a99a6afca..fb597b59fd8 100644 --- a/Tools/pynche/DetailsViewer.py +++ b/Tools/pynche/DetailsViewer.py @@ -26,7 +26,7 @@ other side. Thus if red were at 238 and 25 were added to it, red would have the value 7. - Preseve Distance + Preserve Distance When the increment or decrement would send any of the tied variations out of bounds, all tied variations are wrapped as one, so as to preserve the distance between them. Thus if green and blue were tied, diff --git a/Tools/scripts/findnocoding.py b/Tools/scripts/findnocoding.py index e49fc425ffc..5d932908d36 100755 --- a/Tools/scripts/findnocoding.py +++ b/Tools/scripts/findnocoding.py @@ -32,13 +32,13 @@ def walk_python_files(self, paths, *args, **kwargs): "no sophisticated Python source file search will be done.") -decl_re = re.compile(r"coding[=:]\s*([-\w.]+)") +decl_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)') def get_declaration(line): - match = decl_re.search(line) + match = decl_re.match(line) if match: return match.group(1) - return '' + return b'' def has_correct_encoding(text, codec): try: diff --git a/Tools/scripts/fixnotice.py b/Tools/scripts/fixnotice.py index 0ae4872003a..e613b656722 100755 --- a/Tools/scripts/fixnotice.py +++ b/Tools/scripts/fixnotice.py @@ -2,7 +2,7 @@ """(Ostensibly) fix copyright notices in files. -Actually, this sript will simply replace a block of text in a file from one +Actually, this script will simply replace a block of text in a file from one string to another. It will only do this once though, i.e. not globally throughout the file. It writes a backup file and then does an os.rename() dance for atomicity. diff --git a/Tools/scripts/gprof2html.py b/Tools/scripts/gprof2html.py index cb01c2ce680..76408ca4d6b 100755 --- a/Tools/scripts/gprof2html.py +++ b/Tools/scripts/gprof2html.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python2.3 +#! /usr/bin/env python """Transform gprof(1) output into useful HTML.""" diff --git a/Tools/scripts/ifdef.py b/Tools/scripts/ifdef.py index 2ed7a6667bd..5487f1a74be 100755 --- a/Tools/scripts/ifdef.py +++ b/Tools/scripts/ifdef.py @@ -9,11 +9,11 @@ # options. On standard output it writes a copy of the input file(s) # minus those code sections that are suppressed by the selected # combination of defined/undefined symbols. The #if(n)def/#else/#else -# lines themselfs (if the #if(n)def tests for one of the mentioned +# lines themselves (if the #if(n)def tests for one of the mentioned # names) are removed as well. # Features: Arbitrary nesting of recognized and unrecognized -# preprocesor statements works correctly. Unrecognized #if* commands +# preprocessor statements works correctly. Unrecognized #if* commands # are left in place, so it will never remove too much, only too # little. It does accept whitespace around the '#' character. diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py index fe2e2913f1f..418dd269daf 100755 --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -144,13 +144,13 @@ def docs_modified(file_paths): @status("Misc/ACKS updated", modal=True) def credit_given(file_paths): """Check if Misc/ACKS has been changed.""" - return 'Misc/ACKS' in file_paths + return os.path.join('Misc', 'ACKS') in file_paths @status("Misc/NEWS updated", modal=True) def reported_news(file_paths): """Check if Misc/NEWS has been changed.""" - return 'Misc/NEWS' in file_paths + return os.path.join('Misc', 'NEWS') in file_paths def main(): @@ -158,7 +158,8 @@ def main(): python_files = [fn for fn in file_paths if fn.endswith('.py')] c_files = [fn for fn in file_paths if fn.endswith(('.c', '.h'))] doc_files = [fn for fn in file_paths if fn.startswith('Doc')] - special_files = {'Misc/ACKS', 'Misc/NEWS'} & set(file_paths) + misc_files = {os.path.join('Misc', 'ACKS'), os.path.join('Misc', 'NEWS')}\ + & set(file_paths) # PEP 8 whitespace rules enforcement. normalize_whitespace(python_files) # C rules enforcement. @@ -168,9 +169,9 @@ def main(): # Docs updated. docs_modified(doc_files) # Misc/ACKS changed. - credit_given(special_files) + credit_given(misc_files) # Misc/NEWS changed. - reported_news(special_files) + reported_news(misc_files) # Test suite run and passed. if python_files or c_files: diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py index 7f6f191bbeb..850903a1000 100755 --- a/Tools/scripts/pathfix.py +++ b/Tools/scripts/pathfix.py @@ -135,7 +135,7 @@ def fix(filename): except os.error, msg: err('%s: rename failed (%r)\n' % (filename, msg)) return 1 - # Return succes + # Return success return 0 def fixline(line): diff --git a/add_builtins.py b/add_builtins.py new file mode 100755 index 00000000000..162ed6759db --- /dev/null +++ b/add_builtins.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python +'''Add a set of modules to the list of static Python builtins. + +This will modify the Modules/Setup file, which is created by the Static.make +Makefile. When running `make -f Static.make`, this script will be run +automatically using any arguments passed to that Makefile. If you want to add +modules manually, you can run `make -f Static.make setup` which will build +Modules/Setup and allow you to edit it or run this script one or more times. +When finished, run `make -f Static.make` to build the static interpreter. + +Usage: python add_builtins.py [-s /path/to/script] [module_name] ... +''' +from modulefinder import ModuleFinder +import sys +import os +import os.path as op +import re +import imp +import shutil +import types +import importlib +from Cython.Build import cythonize + + +# module definition lines in Modules/Setup look like this +module_def = re.compile('^[A-Za-z_\.]+ .+\.c') +# put extra builtins in here +extra_module_dir = op.join('Modules', 'extras') +if not op.exists(extra_module_dir): + os.makedirs(extra_module_dir) +# file endings that can be cythonized +cythonizeable_exts = ('.py', '.pyx') +# file endings that can be compiled +compileable_exts = ('.c', '.cpp', 'module.c', 'module.cpp') + +def add_builtins(names, script=None, exclude=None, path=None, + auto_add_deps=False, src_dirs=None, test=False): + if path is None: + paths = ['Lib'] + sys.path + elif isinstance(path, basestring): + paths = [path, 'Lib'] + sys.path + else: + paths = path + + if src_dirs is None: src_dirs = {} + + # if called with a script, find their dependencies and re-run + to_add = set(names) + if script: + module_dir = op.split(script)[0] + paths = [module_dir] + paths + finder = ModuleFinder(path=paths) + finder.run_script(script) + to_add.update(finder.modules.keys()) + return add_builtins(list(to_add), script=None, exclude=exclude, + path=paths, auto_add_deps=False, src_dirs=None) + + if auto_add_deps: + for name in names: + try: + f, module_path, _ = imp.find_module(name, paths) + except KeyboardInterrupt: raise + except: continue + + if any([module_path.endswith(x) for x in ('.py', '.pyc')]): + finder = ModuleFinder(path=paths) + finder.run_script(module_path) + to_add.update(finder.modules.keys()) + + + with open('Modules/Setup', 'r') as setup_file: + lines = [str(x).rstrip('\n') for x in setup_file.readlines()] + + # don't add sys (it's already builtin) or anything explicitly excluded + added = {'sys'} + if exclude: + added.update(exclude) + + + # check each module to see if a commented line is present in Modules/Setup, + # and uncomment + for n, line in enumerate(lines): + line = line.strip() + if not line: continue + + for name in to_add: + if line.startswith('#%s ' % name): + lines[n] = line.lstrip('#') + to_add.remove(name) + added.add(name) + print('** Added %s' % name) + break + + # keep track of uncommented module names in Modules/Setup + if module_def.match(line): + module_name = line.split()[0] + pkg = False + try: + f, module_path, _ = imp.find_module(module_name, paths) + if f is None: pkg = True + except: pkg = False + + if not pkg: + added.add(module_name) + print('** Found existing builtin %s' % module_name) + + # don't try to re-add existing builtins + to_add = set.difference(to_add, added) + + for name in list(to_add): + if name in added: continue + + new_lines = [] + + try: + f, module_path, _ = imp.find_module(name, paths) + except ImportError: + if '.' in name: f = None + else: + raise Exception("** Couldn't find module %s" % name) + continue + + # see if the target file already exists in Modules + search_paths = [op.join(*(search_dir + (name+x,))) + for x in compileable_exts + for search_dir in ( + (), + (name,), + ('extras', name), + ) + if op.exists(op.join(*(('Modules',) + search_dir + (name+x,)))) + ] if f else [] + + if search_paths: + module_file = search_paths[0] + print('** Added %s' % module_file) + else: + # import the target module using this python installation, + # and check the corresponding file + + if f is None: + # add package + pkg = name + print("*** Scanning package %s..." % pkg) + + try: + p = importlib.import_module(pkg) + except: + continue + + def get_submodules(x, yielded=None): + if not yielded: yielded = set() + yield x + yielded.add(x) + for member in dir(x): + member = getattr(x, member) + if isinstance(member, types.ModuleType) and not member in yielded: + for y in get_submodules(member, yielded): + yield y + yielded.add(y) + + submodules = get_submodules(p) + + for submodule in submodules: + name = submodule.__name__ + + sys.stdout.write("** Adding module %s in package %s..." % (name, pkg)) + sys.stdout.flush() + + try: + add = add_module(name, added, paths, src_dirs) + if add: new_lines += add + except Exception as e: + print('Failed:', e) + #raise + + print('done.') + + else: + # add standalone module + sys.stdout.write('** Adding %s...' % name) + sys.stdout.flush() + + try: + add = add_module(name, added, paths, src_dirs, module_path=module_path) + if add: new_lines += add + except Exception as e: + print(e) + + print('done.') + + if new_lines: lines += new_lines + + with open('Modules/Setup', 'w') as setup_file: + setup_file.write('\n'.join(lines)) + + + +def add_module(name, added, paths, src_dirs, module_path=None): + if name in added: + return + + added.add(name) + pkg = '.' in name + opts = '' + + if not module_path: + try: module_path = importlib.import_module(name).__file__ + except: return + + if op.basename(module_path).startswith('__init__'): + pkg = True + + # if it's a .pyc file, hope the original python source is right next to it! + if module_path.endswith('.pyc'): + if op.exists(module_path[:-1]): + module_path = module_path[:-1] + else: + # TODO: this could possibly be handled by unpyclib, etc. + raise Exception('Lone .pyc file %s' % module_path) + + module_dir, module_file = op.split(module_path) + + # copy the file to the Modules/extras directory + dest_dir = extra_module_dir + if pkg: + dest_dir = op.join(dest_dir, name.split('.')[0]) + if not op.exists(dest_dir): os.makedirs(dest_dir) + + + # if it's a shared library, try to find the original C or C++ source file to + # compile into a static library; otherwise, there's nothing we can do here + if module_file.endswith('.so'): + done = False + module_dirs = [] + for k, v in src_dirs.items(): + # if user specified a src directory for this package, + # include it in the module search path + if name.startswith(k): + module = name[len(k):].lstrip('.') + if '.' in module: + v = op.join(v, *module.split('.')) + module_dirs += [v] + module_dirs += [module_dir] + for search_dir in module_dirs + paths: + for compiled_module in ('.'.join(module_file.split('.')[:-1]) + ext + for ext in compileable_exts): + if op.exists(op.join(search_dir, compiled_module)): + dest_file = compiled_module + if pkg: + dest_file = '__'.join(name.split('.')) + '.' + dest_file.split('.')[-1] + + dest_path = op.join(dest_dir, dest_file) + + if not op.exists(dest_path): + shutil.copy(op.join(search_dir, compiled_module), dest_path) + + modile_dir = search_dir + module_file = dest_file + opts += ' -I%s' % op.abspath(search_dir) + done = True; break + + if done: break + + if not any([module_file.endswith(ext) for ext in compileable_exts]): + raise Exception("Couldn't find C source file for %s" % module_file) + #return + + module_path = op.join(module_dir, module_file) + + else: + # copy the file to the Modules/extras directory + dest_file = module_file + if pkg: + dest_file = '__'.join(name.split('.')) + '.' + dest_file.split('.')[-1] + + dest_path = op.join(dest_dir, dest_file) + opts = '' + + if not op.exists(dest_path): + shutil.copy(module_path, dest_path) + + + # if the file ends in .py or .pyx, try to compile with Cython + if any([module_file.endswith(x) for x in cythonizeable_exts]): + dest_file = '.'.join(dest_file.split('.')[:-1]) + '.c' + + if op.exists(op.join(dest_dir, dest_file)): + module_file = dest_file + else: + try: + cythonize(dest_path) + module_file = dest_file + dest_path = op.join(dest_dir, dest_file) + if pkg: + # correct module name in Cython-generated C file + wrong_name = '.'.join(dest_file.split('.')[:-1]) + + with open(dest_path, 'r') as input_file: + with open(dest_path + '2', 'w') as output_file: + for line in input_file: + line = line.replace('"%s"' % wrong_name, '"%s"' % name) + output_file.write(line) + + os.remove(dest_path) + shutil.move(dest_path + '2', dest_path) + + except KeyboardInterrupt: raise + except: + os.remove(op.join(dest_dir, dest_file)) + raise Exception('Cython failed to compile %s' % dest_path) + + if any([module_file.endswith(ext) for ext in compileable_exts]): + # if there's a directory called Modules/{module} or + # Modules/extras/{module}, include that directory when compiling + for inc_dir in (op.join('Modules', name), op.join(dest_dir, name)): + if op.exists(inc_dir) and op.isdir(inc_dir): + opts += ' -I%s' % op.abspath(inc_dir) + + if pkg: + module_file = op.join('extras', + name.split('.')[0], + '.'.join(module_file.split('.')[:-1]).replace( + '.', '__') + '.' + module_file.split('.')[-1] + ) + else: + module_file = op.join('extras', module_file) + + # add a line to Modules/Setup + return ['%s %s%s' % (name.replace('.', '__'), module_file, opts)] + else: + raise Exception('Unknown file: %s' % module_file) + + +if __name__ == '__main__': + import argparse + + parser = argparse.ArgumentParser() + + parser.add_argument('module', nargs='*', help='names of modules to be added as builtins') + parser.add_argument('-s', '--script', nargs='?', default=None, + help='add all dependencies of this script as builtins') + parser.add_argument('-e', '--exclude', nargs='?', default=None, + help='comma-separated list of modules to be excluded') + parser.add_argument('-p', '--path', nargs='?', default=None, + help='add this path to the module search path') + parser.add_argument('-d', '--deps', action='store_true', + help='when adding a module, automatically add all of its dependencies') + parser.add_argument('--src', nargs='?', default=None, + help='list of source package locations for shared libraries, e.g. `pkg1:/path/to/src,pkg2:/path/to/src`') + parser.add_argument('-t', '--test', action='store_true', + help="don't actually add the modules right now, just output their names") + + args = parser.parse_args() + + src_dirs = {pkg.split(':')[0]:pkg.split(':')[1] + for pkg in args.src.split(',') + } if args.src else None + + add_builtins(args.module, + script=args.script, + exclude=args.exclude.split(',') if args.exclude else None, + path=args.path, + auto_add_deps=args.deps, + src_dirs=src_dirs, + test=args.test + ) diff --git a/configure b/configure index 528aa512ae0..68e47842b56 100755 --- a/configure +++ b/configure @@ -2981,6 +2981,7 @@ fi +ARCH_RUN_32BIT="" UNIVERSAL_ARCHS="32-bit" @@ -6253,7 +6254,7 @@ then { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether gcc supports ParseTuple __format__" >&5 $as_echo_n "checking whether gcc supports ParseTuple __format__... " >&6; } save_CFLAGS=$CFLAGS - CFLAGS="$CFLAGS -Werror" + CFLAGS="$CFLAGS -Werror -Wformat" cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -6653,7 +6654,7 @@ sys/param.h sys/poll.h sys/select.h sys/socket.h sys/statvfs.h sys/stat.h \ sys/termio.h sys/time.h \ sys/times.h sys/types.h sys/un.h sys/utsname.h sys/wait.h pty.h libutil.h \ sys/resource.h netpacket/packet.h sysexits.h bluetooth.h \ -bluetooth/bluetooth.h linux/tipc.h spawn.h util.h +bluetooth/bluetooth.h linux/tipc.h spawn.h util.h alloca.h do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" @@ -7996,7 +7997,6 @@ case $ac_sys_system/$ac_sys_release in esac -ARCH_RUN_32BIT="" case $ac_sys_system/$ac_sys_release in Darwin/[01567]\..*) @@ -9627,6 +9627,17 @@ $as_echo "#define HAVE_BROKEN_PTHREAD_SIGMASK 1" >>confdefs.h ;; esac +fi +done + + for ac_func in pthread_atfork +do : + ac_fn_c_check_func "$LINENO" "pthread_atfork" "ac_cv_func_pthread_atfork" +if test "x$ac_cv_func_pthread_atfork" = xyes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_PTHREAD_ATFORK 1 +_ACEOF + fi done diff --git a/configure.ac b/configure.ac index cb2a958fbfe..d922bc7cb17 100644 --- a/configure.ac +++ b/configure.ac @@ -145,6 +145,7 @@ fi AC_SUBST(UNIVERSALSDK) AC_SUBST(ARCH_RUN_32BIT) +ARCH_RUN_32BIT="" UNIVERSAL_ARCHS="32-bit" AC_SUBST(LIPO_32BIT_FLAGS) @@ -1326,7 +1327,7 @@ if test "$GCC" = "yes" then AC_MSG_CHECKING(whether gcc supports ParseTuple __format__) save_CFLAGS=$CFLAGS - CFLAGS="$CFLAGS -Werror" + CFLAGS="$CFLAGS -Werror -Wformat" AC_COMPILE_IFELSE([ AC_LANG_PROGRAM([[void f(char*,...)__attribute((format(PyArg_ParseTuple, 1, 2)));]], [[]]) ],[ @@ -1518,7 +1519,7 @@ sys/param.h sys/poll.h sys/select.h sys/socket.h sys/statvfs.h sys/stat.h \ sys/termio.h sys/time.h \ sys/times.h sys/types.h sys/un.h sys/utsname.h sys/wait.h pty.h libutil.h \ sys/resource.h netpacket/packet.h sysexits.h bluetooth.h \ -bluetooth/bluetooth.h linux/tipc.h spawn.h util.h) +bluetooth/bluetooth.h linux/tipc.h spawn.h util.h alloca.h) AC_HEADER_DIRENT AC_HEADER_MAJOR @@ -1801,7 +1802,6 @@ case $ac_sys_system/$ac_sys_release in esac -ARCH_RUN_32BIT="" AC_SUBST(LIBTOOL_CRUFT) case $ac_sys_system/$ac_sys_release in Darwin/@<:@01567@:>@\..*) @@ -2569,6 +2569,7 @@ if test "$posix_threads" = "yes"; then [Define if pthread_sigmask() does not work on your system.]) ;; esac]) + AC_CHECK_FUNCS(pthread_atfork) fi diff --git a/pyconfig.h.in b/pyconfig.h.in index 231b9c8a31f..8ac017e2383 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -55,6 +55,9 @@ /* Define to 1 if you have the `alarm' function. */ #undef HAVE_ALARM +/* Define to 1 if you have the header file. */ +#undef HAVE_ALLOCA_H + /* Define this if your time.h defines altzone. */ #undef HAVE_ALTZONE @@ -517,6 +520,9 @@ /* Define if you have GNU PTH threads. */ #undef HAVE_PTH +/* Define to 1 if you have the `pthread_atfork' function. */ +#undef HAVE_PTHREAD_ATFORK + /* Defined for Solaris 2.6 bug in pthread header. */ #undef HAVE_PTHREAD_DESTRUCTOR diff --git a/setup.py b/setup.py index ea8a5f51e96..8d8ec84d814 100644 --- a/setup.py +++ b/setup.py @@ -437,8 +437,11 @@ def add_gcc_paths(self): def detect_modules(self): # Ensure that /usr/local is always used - add_dir_to_list(self.compiler.library_dirs, '/usr/local/lib') - add_dir_to_list(self.compiler.include_dirs, '/usr/local/include') + if not cross_compiling: + add_dir_to_list(self.compiler.library_dirs, '/usr/local/lib') + add_dir_to_list(self.compiler.include_dirs, '/usr/local/include') + if cross_compiling: + self.add_gcc_paths() self.add_multiarch_paths() # Add paths specified in the environment variables LDFLAGS and diff --git a/static_freeze.py b/static_freeze.py new file mode 100755 index 00000000000..2a3c2b2f7b0 --- /dev/null +++ b/static_freeze.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python +'''This script generates, runs, and then deletes a Makefile which will compile a +Python script into a standalone executable which is statically linked to the +Python runtime. + +To use this script, first generate a static Python library (see the parent +directory's README.md). + +Usage: + + python static_freeze.py test_file.py /path/to/static/libpythonX.X.a + +Any additional arguments will be passed to the generated Makefile, +e.g. PYTHON=python3 +''' + +import sys +import os +from subprocess import call +major_version = sys.version_info[0] + + +make_template = '''# Makefile for creating our standalone Cython program +PYTHON=./python +CYTHON=cython +PYVERSION=$(shell $(PYTHON) -c "import sys; print(sys.version[:3])") + +#INCDIR=$(shell $(PYTHON) -c "from distutils import sysconfig; print(sysconfig.get_python_inc())") +INCDIR=Include/ +PLATINCDIR=$(shell $(PYTHON) -c "from distutils import sysconfig; print(sysconfig.get_python_inc(plat_specific=True))") +LIBDIR1=$(shell $(PYTHON) -c "from distutils import sysconfig; print(sysconfig.get_config_var('LIBDIR'))") +LIBDIR2=%library_dir% +PYLIB=%library_name% + +CC=$(shell $(PYTHON) -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('CC'))") +LINKCC=$(shell $(PYTHON) -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('LINKCC'))") +LIBS=$(shell $(PYTHON) -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('LIBS'))") +SYSLIBS= $(shell $(PYTHON) -c "import distutils.sysconfig; print(distutils.sysconfig.get_config_var('SYSLIBS'))") + +INCLUDE= + +%name%: %name%.o %library_dir%/lib%library_name%.a + $(LINKCC) -o $@ $^ -L$() -L$(LIBDIR1) -L$(LIBDIR2) -l$(PYLIB) $(LIBS) $(SYSLIBS) $(INCLUDE) + +%name%.o: %name%.c + $(CC) -c $^ -I$(INCDIR) -I$(PLATINCDIR) $(INCLUDE) + +%name%.c: %filename% + @$(CYTHON) -%major% --embed %filename% + +all: %name% + +clean: + rm -f %name% %name%.c %name%.o +''' + +def freeze(filename, library, make_args=None): + if make_args is None: make_args = [] + + name = '.'.join(filename.split('.')[:-1]) + library_dir, library_name = os.path.split(os.path.abspath(library)) + library_name = '.'.join((library.split('/')[-1][3:]).split('.')[:-1]) + + template = make_template + # generate makefile + for a, b in ( + ('%name%', name), + ('%filename%', filename), + ('%library_dir%', library_dir), + ('%library_name%', library_name), + ('%major%', str(major_version)), + ): + template = template.replace(a, b) + + with open(filename + '.make', 'wb') as make_file: + make_file.write(bytes(template, 'utf8')) + + # call make + call(['make', '-f', '%s.make' % filename] + make_args) + + # delete makefile + os.remove(filename + '.make') + + +if __name__ == '__main__': + def fail(message): + print(__doc__) + print(message) + sys.exit() + + try: + script_file = sys.argv[1] + assert os.path.exists(script_file) + except IndexError: + fail('ERROR: No script specified') + except AssertionError: + fail('ERROR: Script not found') + + try: + lib_path = sys.argv[2] + assert os.path.exists(lib_path) + except IndexError: + fail('ERROR: Path to Python runtime not specified') + except AssertionError: + fail('ERROR: Python runtime not found') + + try: make_args = sys.argv[3:] + except IndexError: make_args = [] + + freeze(script_file, lib_path, make_args=make_args) \ No newline at end of file