Skip to content

gh-120754: Refactor I/O modules to stash whole stat result rather than individual members #123412

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vstinner merged 7 commits into from
Sep 18, 2024
Merged

gh-120754: Refactor I/O modules to stash whole stat result rather than individual members #123412

Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
508aa9d
gh-120754: Refactor _io to stash whole stat
cmaloney Aug 27, 2024
9d849ce
gh-120754: Refactor _pyio to stash whole stat
cmaloney Aug 27, 2024
bfcfcf2
Apply suggestions from code review
cmaloney Aug 29, 2024
3122665
Add comments around stat_atopen and why bufsize is + 1
cmaloney Aug 29, 2024
8f5cfe4
Apply review changes for _pyio _blkszie
cmaloney Aug 29, 2024
d18a82d
Fix comment formatting
cmaloney Aug 29, 2024
c55d10e
Add _pyiio +1 comment to fileio for better clarity
cmaloney Aug 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
gh-120754: Refactor _io to stash whole stat 
Multiple places in the I/O stack optimize common cases by using the
information from stat. Currently individual members are extracted from
the stat and stored into the fileio struct. Refactor the code to store
the whole stat struct instead.
  • Loading branch information
@cmaloney
cmaloney committed Aug 27, 2024
commit 508aa9d23e16a4a358ce5fc978de3c82d7602809
67 changes: 45 additions & 22 deletions Modules/_io/fileio.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,7 @@ typedef struct {
signed int seekable : 2; /* -1 means unknown */
unsigned int closefd : 1;
char finalizing;
unsigned int blksize;
Py_off_t estimated_size;
struct _Py_stat_struct *stat_atopen;
Comment thread
vstinner marked this conversation as resolved.
PyObject *weakreflist;
PyObject *dict;
} fileio;
Expand Down Expand Up @@ -199,8 +198,7 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
self->writable = 0;
self->appending = 0;
self->seekable = -1;
self->blksize = 0;
self->estimated_size = -1;
self->stat_atopen = NULL;
self->closefd = 1;
self->weakreflist = NULL;
}
Expand Down Expand Up @@ -256,7 +254,6 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode,
#elif !defined(MS_WINDOWS)
int *atomic_flag_works = NULL;
#endif
struct _Py_stat_struct fdfstat;
int fstat_result;
int async_err = 0;

Expand Down Expand Up @@ -454,9 +451,13 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode,
#endif
}

self->blksize = DEFAULT_BUFFER_SIZE;
self->stat_atopen = PyMem_New(struct _Py_stat_struct, 1);
if (self->stat_atopen == NULL) {
PyErr_Format(PyExc_MemoryError, "Unable to allocate space for stat result");
Comment thread
cmaloney marked this conversation as resolved.
Outdated
goto error;
}
Py_BEGIN_ALLOW_THREADS
fstat_result = _Py_fstat_noraise(self->fd, &fdfstat);
fstat_result = _Py_fstat_noraise(self->fd, self->stat_atopen);
Py_END_ALLOW_THREADS
if (fstat_result < 0) {
/* Tolerate fstat() errors other than EBADF. See Issue #25717, where
Expand All @@ -471,25 +472,21 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode,
#endif
goto error;
}

PyMem_Free(self->stat_atopen);
self->stat_atopen = NULL;
}
else {
#if defined(S_ISDIR) && defined(EISDIR)
/* On Unix, open will succeed for directories.
In Python, there should be no file objects referring to
directories, so we need a check. */
if (S_ISDIR(fdfstat.st_mode)) {
if (S_ISDIR(self->stat_atopen->st_mode)) {
errno = EISDIR;
PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, nameobj);
goto error;
}
#endif /* defined(S_ISDIR) */
#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE
if (fdfstat.st_blksize > 1)
self->blksize = fdfstat.st_blksize;
#endif /* HAVE_STRUCT_STAT_ST_BLKSIZE */
if (fdfstat.st_size < PY_SSIZE_T_MAX) {
self->estimated_size = (Py_off_t)fdfstat.st_size;
}
}

#if defined(MS_WINDOWS) || defined(__CYGWIN__)
Expand Down Expand Up @@ -521,6 +518,10 @@ _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode,
internal_close(self);
_PyErr_ChainExceptions1(exc);
}
if (self->stat_atopen != NULL) {
PyMem_Free(self->stat_atopen);
self->stat_atopen = NULL;
}

done:
#ifdef MS_WINDOWS
Expand Down Expand Up @@ -553,6 +554,10 @@ fileio_dealloc(fileio *self)
if (_PyIOBase_finalize((PyObject *) self) < 0)
return;
_PyObject_GC_UNTRACK(self);
if (self->stat_atopen != NULL) {
PyMem_Free(self->stat_atopen);
self->stat_atopen = NULL;
}
if (self->weakreflist != NULL)
PyObject_ClearWeakRefs((PyObject *) self);
(void)fileio_clear(self);
Expand Down Expand Up @@ -725,7 +730,12 @@ _io_FileIO_readall_impl(fileio *self)
return err_closed();
}

end = self->estimated_size;
if (self->stat_atopen != NULL && self->stat_atopen->st_size < _PY_READ_MAX) {
end = (Py_off_t)self->stat_atopen->st_size;
}
else {
end = -1;
}
if (end <= 0) {
/* Use a default size and resize as needed. */
bufsize = SMALLCHUNK;
Expand Down Expand Up @@ -1094,11 +1104,13 @@ _io_FileIO_truncate_impl(fileio *self, PyTypeObject *cls, PyObject *posobj)
return NULL;
}

/* Sometimes a large file is truncated. While estimated_size is used as a
estimate, that it is much larger than the actual size can result in a
significant over allocation and sometimes a MemoryError / running out of
memory. */
self->estimated_size = pos;
/* Since the file was truncated, its size at open is no longer accurate
as an estimate. Clear out the stat result, and rely on dynamic resize
Comment thread
vstinner marked this conversation as resolved.
code if a readall is requested. */
if (self->stat_atopen != NULL) {
PyMem_Free(self->stat_atopen);
self->stat_atopen = NULL;
}

return posobj;
}
Expand Down Expand Up @@ -1229,16 +1241,27 @@ get_mode(fileio *self, void *closure)
return PyUnicode_FromString(mode_string(self));
}

static PyObject *
get_blksize(fileio *self, void *closure)
{
#ifdef HAVE_STRUCT_STAT_ST_BLKSIZE
if (self->stat_atopen != NULL && self->stat_atopen->st_blksize > 1) {

@gpshead gpshead Sep 18, 2024

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I do wonder how realistic the st_blksize values, when available, are for performance purposes, I guess we'll find out.

@vstinner vstinner Sep 18, 2024

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This PR should not change the buffer size, does it?

@cmaloney cmaloney Sep 18, 2024

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#117151 (comment) investigated st_blksize a bit previously. This PR I tried not to change buffer size at all / just change how it is accessed.

Have with the refactors + optimizations been watching for new issues. Are finding some as people test main (ex. gh-113977 which I wrote a primary fix for #122101, and have more fix ideas on top of the stat_atopen changes)

return PyLong_FromLong(self->stat_atopen->st_blksize);
}
#endif /* HAVE_STRUCT_STAT_ST_BLKSIZE */
return PyLong_FromLong(DEFAULT_BUFFER_SIZE);
}

static PyGetSetDef fileio_getsetlist[] = {
{"closed", (getter)get_closed, NULL, "True if the file is closed"},
{"closefd", (getter)get_closefd, NULL,
"True if the file descriptor will be closed by close()."},
{"mode", (getter)get_mode, NULL, "String giving the file mode"},
{"_blksize", (getter)get_blksize, NULL, "Stat st_blksize if available"},
{NULL},
};

static PyMemberDef fileio_members[] = {
{"_blksize", Py_T_UINT, offsetof(fileio, blksize), 0},
{"_finalizing", Py_T_BOOL, offsetof(fileio, finalizing), 0},
{"__weaklistoffset__", Py_T_PYSSIZET, offsetof(fileio, weakreflist), Py_READONLY},
{"__dictoffset__", Py_T_PYSSIZET, offsetof(fileio, dict), Py_READONLY},
Expand Down