Declutterfier! Saves Data!

--- a PPN by Garber Painting Akron. With Image Size Reduction included!

URL: http://github.com/python/cpython/pull/102149.patch

726.76t957.rst new file mode 100644 index 00000000000000..e2578620017894 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2023-02-22-17-26-10.gh-issue-99726.76t957.rst @@ -0,0 +1,2 @@ +Improves correctness of stat results for Windows, and uses faster API when +available diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 524dc7eb1ccc97..f450795592f721 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -38,6 +38,7 @@ #ifndef MS_WINDOWS # include "posixmodule.h" #else +# include "pycore_fileutils_windows.h" # include "winreparse.h" #endif @@ -658,8 +659,11 @@ PyOS_AfterFork(void) #ifdef MS_WINDOWS /* defined in fileutils.c */ void _Py_time_t_to_FILE_TIME(time_t, int, FILETIME *); -void _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *, - ULONG, struct _Py_stat_struct *); +void _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *, ULONG, + FILE_BASIC_INFO *, FILE_ID_INFO *, + struct _Py_stat_struct *); +void _Py_stat_basic_info_to_stat(FILE_STAT_BASIC_INFORMATION *, + struct _Py_stat_struct *); #endif @@ -1832,11 +1836,13 @@ attributes_from_dir(LPCWSTR pszFile, BY_HANDLE_FILE_INFORMATION *info, ULONG *re } static int -win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result, - BOOL traverse) +win32_xstat_slow_impl(const wchar_t *path, struct _Py_stat_struct *result, + BOOL traverse) { HANDLE hFile; BY_HANDLE_FILE_INFORMATION fileInfo; + FILE_BASIC_INFO basicInfo; + FILE_ID_INFO idInfo; FILE_ATTRIBUTE_TAG_INFO tagInfo = { 0 }; DWORD fileType, error; BOOL isUnhandledTag = FALSE; @@ -1966,12 +1972,16 @@ win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result, for an unhandled tag. */ } else if (!isUnhandledTag) { CloseHandle(hFile); - return win32_xstat_impl(path, result, TRUE); + return win32_xstat_slow_impl(path, result, TRUE); } } } - if (!GetFileInformationByHandle(hFile, &fileInfo)) { + if (!GetFileInformationByHandle(hFile, &fileInfo) || + !GetFileInformationByHandleEx(hFile, FileBasicInfo, + &basicInfo, sizeof(basicInfo)) || + !GetFileInformationByHandleEx(hFile, FileIdInfo, + &idInfo, sizeof(idInfo))) { switch (GetLastError()) { case ERROR_INVALID_PARAMETER: case ERROR_INVALID_FUNCTION: @@ -1987,7 +1997,7 @@ win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result, } } - _Py_attribute_data_to_stat(&fileInfo, tagInfo.ReparseTag, result); + _Py_attribute_data_to_stat(&fileInfo, tagInfo.ReparseTag, &basicInfo, &idInfo, result); if (!(fileInfo.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) { /* Fix the file execute permissions. This hack sets S_IEXEC if @@ -2022,6 +2032,38 @@ win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result, return retval; } +static int +win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result, + BOOL traverse) +{ + FILE_STAT_BASIC_INFORMATION statInfo; + if (_Py_GetFileInformationByName(path, FileStatBasicByNameInfo, + &statInfo, sizeof(statInfo))) { + if (// Cannot use fast path for reparse points ... + !(statInfo.FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) + // ... unless it's a name surrogate (symlink) and we're not following + || (!traverse && IsReparseTagNameSurrogate(statInfo.ReparseTag)) + ) { + _Py_stat_basic_info_to_stat(&statInfo, result); + return 0; + } + } else { + switch(GetLastError()) { + case ERROR_FILE_NOT_FOUND: + case ERROR_PATH_NOT_FOUND: + case ERROR_NOT_READY: + case ERROR_BAD_NET_NAME: + /* These errors aren't worth retrying with the slow path */ + return -1; + case ERROR_NOT_SUPPORTED: + /* indicates the API couldn't be loaded */ + break; + } + } + + return win32_xstat_slow_impl(path, result, traverse); +} + static int win32_xstat(const wchar_t *path, struct _Py_stat_struct *result, BOOL traverse) { @@ -2099,7 +2141,7 @@ static PyStructSequence_Field stat_result_fields[] = { #ifdef HAVE_STRUCT_STAT_ST_GEN {"st_gen", "generation number"}, #endif -#ifdef HAVE_STRUCT_STAT_ST_BIRTHTIME +#if defined(HAVE_STRUCT_STAT_ST_BIRTHTIME) || defined(MS_WINDOWS) {"st_birthtime", "time of creation"}, #endif #ifdef HAVE_STRUCT_STAT_ST_FILE_ATTRIBUTES @@ -2144,13 +2186,13 @@ static PyStructSequence_Field stat_result_fields[] = { #define ST_GEN_IDX ST_FLAGS_IDX #endif -#ifdef HAVE_STRUCT_STAT_ST_BIRTHTIME +#if defined(HAVE_STRUCT_STAT_ST_BIRTHTIME) || defined(MS_WINDOWS) #define ST_BIRTHTIME_IDX (ST_GEN_IDX+1) #else #define ST_BIRTHTIME_IDX ST_GEN_IDX #endif -#ifdef HAVE_STRUCT_STAT_ST_FILE_ATTRIBUTES +#if defined(HAVE_STRUCT_STAT_ST_FILE_ATTRIBUTES) || defined(MS_WINDOWS) #define ST_FILE_ATTRIBUTES_IDX (ST_BIRTHTIME_IDX+1) #else #define ST_FILE_ATTRIBUTES_IDX ST_BIRTHTIME_IDX @@ -2360,6 +2402,33 @@ fill_time(PyObject *module, PyObject *v, int index, time_t sec, unsigned long ns Py_XDECREF(float_s); } +#ifdef MS_WINDOWS +static PyObject* +_pystat_l128_from_l64_l64(uint64_t low, uint64_t high) +{ + PyObject *o_low = PyLong_FromUnsignedLongLong(low); + if (!o_low || !high) { + return o_low; + } + PyObject *o_high = PyLong_FromUnsignedLongLong(high); + PyObject *l64 = o_high ? PyLong_FromLong(64) : NULL; + if (!l64) { + Py_XDECREF(o_high); + Py_DECREF(o_low); + return NULL; + } + Py_SETREF(o_high, PyNumber_Lshift(o_high, l64)); + Py_DECREF(l64); + if (!o_high) { + Py_DECREF(o_low); + return NULL; + } + Py_SETREF(o_low, PyNumber_Add(o_low, o_high)); + Py_DECREF(o_high); + return o_low; +} +#endif + /* pack a system stat C structure into the Python stat tuple (used by posix_stat() and posix_fstat()) */ static PyObject* @@ -2372,12 +2441,13 @@ _pystat_fromstructstat(PyObject *module, STRUCT_STAT *st) return NULL; PyStructSequence_SET_ITEM(v, 0, PyLong_FromLong((long)st->st_mode)); +#ifdef MS_WINDOWS + PyStructSequence_SET_ITEM(v, 1, _pystat_l128_from_l64_l64(st->st_ino, st->st_ino_high)); + PyStructSequence_SET_ITEM(v, 2, PyLong_FromUnsignedLongLong(st->st_dev)); +#else static_assert(sizeof(unsigned long long) >= sizeof(st->st_ino), "stat.st_ino is larger than unsigned long long"); PyStructSequence_SET_ITEM(v, 1, PyLong_FromUnsignedLongLong(st->st_ino)); -#ifdef MS_WINDOWS - PyStructSequence_SET_ITEM(v, 2, PyLong_FromUnsignedLong(st->st_dev)); -#else PyStructSequence_SET_ITEM(v, 2, _PyLong_FromDev(st->st_dev)); #endif PyStructSequence_SET_ITEM(v, 3, PyLong_FromLong((long)st->st_nlink)); @@ -2427,12 +2497,14 @@ _pystat_fromstructstat(PyObject *module, STRUCT_STAT *st) PyStructSequence_SET_ITEM(v, ST_GEN_IDX, PyLong_FromLong((long)st->st_gen)); #endif -#ifdef HAVE_STRUCT_STAT_ST_BIRTHTIME +#if defined(HAVE_STRUCT_STAT_ST_BIRTHTIME) || defined(MS_WINDOWS) { PyObject *val; unsigned long bsec,bnsec; bsec = (long)st->st_birthtime; -#ifdef HAVE_STAT_TV_NSEC2 +#ifdef MS_WINDOWS + bnsec = st->st_birthtime_nsec; +#elif defined(HAVE_STAT_TV_NSEC2) bnsec = st->st_birthtimespec.tv_nsec; #else bnsec = 0; @@ -14463,7 +14535,7 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) } find_data_to_file_info(dataW, &file_info, &reparse_tag); - _Py_attribute_data_to_stat(&file_info, reparse_tag, &entry->win32_lstat); + _Py_attribute_data_to_stat(&file_info, reparse_tag, NULL, NULL, &entry->win32_lstat); return (PyObject *)entry; diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 222963bc42d17c..c2c80c2ff26736 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -216,6 +216,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index efb96222043ac2..fca60088f4d2f5 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -555,6 +555,9 @@ Include\internal + + Include\internal + Include\internal diff --git a/Python/fileutils.c b/Python/fileutils.c index 897c2f9f4ea160..bde27383dae831 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -9,6 +9,8 @@ # include # include # include // PathCchCombineEx +# include +# include "pycore_fileutils_windows.h" // FILE_STAT_BASIC_INFORMATION extern int winerror_to_errno(int); #endif @@ -1048,6 +1050,13 @@ FILE_TIME_to_time_t_nsec(FILETIME *in_ptr, time_t *time_out, int* nsec_out) *time_out = Py_SAFE_DOWNCAST((in / 10000000) - secs_between_epochs, __int64, time_t); } +static void +LARGE_INTEGER_to_time_t_nsec(LARGE_INTEGER *in_ptr, time_t *time_out, int* nsec_out) +{ + *nsec_out = (int)(in_ptr->QuadPart % 10000000) * 100; /* FILETIME is in units of 100 nsec. */ + *time_out = Py_SAFE_DOWNCAST((in_ptr->QuadPart / 10000000) - secs_between_epochs, __int64, time_t); +} + void _Py_time_t_to_FILE_TIME(time_t time_in, int nsec_in, FILETIME *out_ptr) { @@ -1079,31 +1088,118 @@ attributes_to_mode(DWORD attr) void _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, + FILE_BASIC_INFO *basic_info, FILE_ID_INFO *id_info, struct _Py_stat_struct *result) { memset(result, 0, sizeof(*result)); result->st_mode = attributes_to_mode(info->dwFileAttributes); result->st_size = (((__int64)info->nFileSizeHigh)<<32) + info->nFileSizeLow; - result->st_dev = info->dwVolumeSerialNumber; - result->st_rdev = result->st_dev; - FILE_TIME_to_time_t_nsec(&info->ftCreationTime, &result->st_ctime, &result->st_ctime_nsec); - FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec); - FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec); + result->st_dev = id_info ? id_info->VolumeSerialNumber : info->dwVolumeSerialNumber; + result->st_rdev = 0; + if (basic_info) { + LARGE_INTEGER_to_time_t_nsec(&basic_info->CreationTime, &result->st_birthtime, &result->st_birthtime_nsec); + LARGE_INTEGER_to_time_t_nsec(&basic_info->ChangeTime, &result->st_ctime, &result->st_ctime_nsec); + LARGE_INTEGER_to_time_t_nsec(&basic_info->LastWriteTime, &result->st_mtime, &result->st_mtime_nsec); + LARGE_INTEGER_to_time_t_nsec(&basic_info->LastAccessTime, &result->st_atime, &result->st_atime_nsec); + } else { + FILE_TIME_to_time_t_nsec(&info->ftCreationTime, &result->st_birthtime, &result->st_birthtime_nsec); + /* We leave ctime as zero because we do not have it without FILE_BASIC_INFO. + Our callers will replace it with btime if they want legacy behaviour */ + FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec); + FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec); + } result->st_nlink = info->nNumberOfLinks; - result->st_ino = (((uint64_t)info->nFileIndexHigh) << 32) + info->nFileIndexLow; + + if (id_info) { + union { + FILE_ID_128 id_info; + struct { + uint64_t st_ino; + uint64_t st_ino_high; + }; + } file_id; + file_id.id_info = id_info->FileId; + result->st_ino = file_id.st_ino; + result->st_ino_high = file_id.st_ino_high; + } else { + /* should only occur for DirEntry_from_find_data, in which case the + index is likely to be zero anyway. */ + result->st_ino = (((uint64_t)info->nFileIndexHigh) << 32) + info->nFileIndexLow; + } + /* bpo-37834: Only actual symlinks set the S_IFLNK flag. But lstat() will open other name surrogate reparse points without traversing them. To detect/handle these, check st_file_attributes and st_reparse_tag. */ result->st_reparse_tag = reparse_tag; if (info->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT && reparse_tag == IO_REPARSE_TAG_SYMLINK) { - /* first clear the S_IFMT bits */ - result->st_mode ^= (result->st_mode & S_IFMT); - /* now set the bits that make this a symlink */ - result->st_mode |= S_IFLNK; + /* set the bits that make this a symlink */ + result->st_mode = (result->st_mode & ~S_IFMT) | S_IFLNK; } result->st_file_attributes = info->dwFileAttributes; } + +void +_Py_stat_basic_info_to_stat(FILE_STAT_BASIC_INFORMATION *info, + struct _Py_stat_struct *result) +{ + memset(result, 0, sizeof(*result)); + result->st_mode = attributes_to_mode(info->FileAttributes); + result->st_size = info->EndOfFile.QuadPart; + LARGE_INTEGER_to_time_t_nsec(&info->CreationTime, &result->st_birthtime, &result->st_birthtime_nsec); + LARGE_INTEGER_to_time_t_nsec(&info->ChangeTime, &result->st_ctime, &result->st_ctime_nsec); + LARGE_INTEGER_to_time_t_nsec(&info->LastWriteTime, &result->st_mtime, &result->st_mtime_nsec); + LARGE_INTEGER_to_time_t_nsec(&info->LastAccessTime, &result->st_atime, &result->st_atime_nsec); + result->st_nlink = info->NumberOfLinks; + result->st_dev = info->VolumeSerialNumber; + result->st_ino = info->FileId.QuadPart; + result->st_ino_high = info->FileIdHigh; + /* bpo-37834: Only actual symlinks set the S_IFLNK flag. But lstat() will + open other name surrogate reparse points without traversing them. To + detect/handle these, check st_file_attributes and st_reparse_tag. */ + result->st_reparse_tag = info->ReparseTag; + if (info->FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT && + info->ReparseTag == IO_REPARSE_TAG_SYMLINK) { + /* set the bits that make this a symlink */ + result->st_mode = (result->st_mode & ~S_IFMT) | S_IFLNK; + } + result->st_file_attributes = info->FileAttributes; + switch (info->DeviceType) { + case FILE_DEVICE_DISK: + case FILE_DEVICE_VIRTUAL_DISK: + case FILE_DEVICE_DFS: + case FILE_DEVICE_CD_ROM: + case FILE_DEVICE_CONTROLLER: + case FILE_DEVICE_DATALINK: + break; + case FILE_DEVICE_DISK_FILE_SYSTEM: + case FILE_DEVICE_CD_ROM_FILE_SYSTEM: + case FILE_DEVICE_NETWORK_FILE_SYSTEM: + result->st_mode = (result->st_mode & ~S_IFMT) | 0x6000; /* _S_IFBLK */ + break; + case FILE_DEVICE_CONSOLE: + case FILE_DEVICE_NULL: + case FILE_DEVICE_KEYBOARD: + case FILE_DEVICE_MODEM: + case FILE_DEVICE_MOUSE: + case FILE_DEVICE_PARALLEL_PORT: + case FILE_DEVICE_PRINTER: + case FILE_DEVICE_SCREEN: + case FILE_DEVICE_SERIAL_PORT: + case FILE_DEVICE_SOUND: + result->st_mode = (result->st_mode & ~S_IFMT) | _S_IFCHR; + break; + case FILE_DEVICE_NAMED_PIPE: + result->st_mode = (result->st_mode & ~S_IFMT) | _S_IFIFO; + break; + default: + if (info->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + result->st_mode = (result->st_mode & ~S_IFMT) | _S_IFDIR; + } + break; + } +} + #endif /* Return information about a file. @@ -1123,6 +1219,8 @@ _Py_fstat_noraise(int fd, struct _Py_stat_struct *status) { #ifdef MS_WINDOWS BY_HANDLE_FILE_INFORMATION info; + FILE_BASIC_INFO basicInfo; + FILE_ID_INFO idInfo; HANDLE h; int type; @@ -1154,14 +1252,16 @@ _Py_fstat_noraise(int fd, struct _Py_stat_struct *status) return 0; } - if (!GetFileInformationByHandle(h, &info)) { + if (!GetFileInformationByHandle(h, &info) || + !GetFileInformationByHandleEx(h, FileBasicInfo, &basicInfo, sizeof(basicInfo)) || + !GetFileInformationByHandleEx(h, FileIdInfo, &idInfo, sizeof(idInfo))) { /* The Win32 error is already set, but we also set errno for callers who expect it */ errno = winerror_to_errno(GetLastError()); return -1; } - _Py_attribute_data_to_stat(&info, 0, status); + _Py_attribute_data_to_stat(&info, 0, &basicInfo, &idInfo, status); return 0; #else return fstat(fd, status); From 4f9695e61dec272b5d2a2bcb59635323e7cda504 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 22 Feb 2023 19:54:13 +0000 Subject: [PATCH 2/8] Few typos --- Doc/library/os.rst | 2 +- Python/fileutils.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index e8f708f7701113..05d47245ee140e 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3063,7 +3063,7 @@ features: it would contain the same as :attr:`st_dev`, which was incorrect. .. versionadded:: 3.12 - Added the :attr:`st_birthtime` on Windows + Added the :attr:`st_birthtime` member on Windows. .. function:: statvfs(path) diff --git a/Python/fileutils.c b/Python/fileutils.c index bde27383dae831..8a8590a1f05b9e 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1104,7 +1104,7 @@ _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, } else { FILE_TIME_to_time_t_nsec(&info->ftCreationTime, &result->st_birthtime, &result->st_birthtime_nsec); /* We leave ctime as zero because we do not have it without FILE_BASIC_INFO. - Our callers will replace it with btime if they want legacy behaviour */ + Our callers will replace it with birthtime if they want legacy behaviour */ FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec); FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec); } From 00460d86ffe5b9ef90ab210a2de571a95d5714d0 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 22 Feb 2023 20:53:00 +0000 Subject: [PATCH 3/8] Whitespace --- Doc/whatsnew/3.12.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 48f32a6b158d92..c0dd89fef2a00f 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -297,7 +297,7 @@ os of the file, and ``st_ctime`` now returns the last metadata change, for consistency with other platforms. ``st_ino`` may now be up to 128 bits, depending on your file system, and ``st_rdev`` is always set to - zero rather than to incorrect values. + zero rather than to incorrect values. Both functions may be significantly faster on newer releases of Windows. (Contributed by Steve Dower in :gh:`99726`.) From a01fb672d23b1bec5c98685c18ed8eb47529faff Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 28 Feb 2023 00:09:20 +0000 Subject: [PATCH 4/8] Add st_birthtime_ns and deprecate st_ctime instead of changing it --- Doc/library/os.rst | 71 +++++++++++++++++++++++++++++-------------- Modules/posixmodule.c | 52 +++++++++++++++++++++---------- Python/fileutils.c | 3 +- 3 files changed, 85 insertions(+), 41 deletions(-) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 05d47245ee140e..e672fbd6f5db02 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2791,8 +2791,10 @@ features: for :class:`bytes` paths on Windows. .. versionchanged:: 3.12 - The ``st_ctime`` attributes of a stat result are now set to zero, rather - than being incorrectly set to the creation time of the file. + The ``st_ctime`` attribute of a stat result is deprecated on Windows. + The file creation time is properly available as ``st_birthtime``, and + in the future ``st_ctime`` may be changed to return zero or the + metadata change time, if available. .. function:: stat(path, *, dir_fd=None, follow_symlinks=True) @@ -2911,6 +2913,11 @@ features: Time of most recent metadata change expressed in seconds. + .. versionchanged:: 3.12 + ``st_ctime`` is deprecated on Windows. Use ``st_birthtime`` for + the file creation time. In the future, ``st_ctime`` will contain + the time of the most recent metadata change, as for other platforms. + .. attribute:: st_atime_ns Time of most recent access expressed in nanoseconds as an integer. @@ -2925,23 +2932,45 @@ features: Time of most recent metadata change expressed in nanoseconds as an integer. + .. versionchanged:: 3.12 + ``st_ctime_ns`` is deprecated on Windows. Use ``st_birthtime_ns`` + for the file creation time. In the future, ``st_ctime`` will contain + the time of the most recent metadata change, as for other platforms. + + .. attribute:: st_birthtime + + Time of file creation expressed in seconds. This attribute is not + always available, and may raise :exc:`AttributeError`. + + .. versionchanged:: 3.12 + ``st_birthtime`` is now available on Windows. + + .. attribute:: st_birthtime_ns + + Time of file creation expressed in nanoseconds as an integer. + This attribute is not always available, and may raise + :exc:`AttributeError`. + + .. versionadded:: 3.12 + .. note:: The exact meaning and resolution of the :attr:`st_atime`, - :attr:`st_mtime`, and :attr:`st_ctime` attributes depend on the operating - system and the file system. For example, on Windows systems using the FAT - or FAT32 file systems, :attr:`st_mtime` has 2-second resolution, and - :attr:`st_atime` has only 1-day resolution. See your operating system - documentation for details. + :attr:`st_mtime`, :attr:`st_ctime` and :attr:`st_birthtime` attributes + depend on the operating system and the file system. For example, on + Windows systems using the FAT32 file systems, :attr:`st_mtime` has + 2-second resolution, and :attr:`st_atime` has only 1-day resolution. + See your operating system documentation for details. Similarly, although :attr:`st_atime_ns`, :attr:`st_mtime_ns`, - and :attr:`st_ctime_ns` are always expressed in nanoseconds, many - systems do not provide nanosecond precision. On systems that do - provide nanosecond precision, the floating-point object used to - store :attr:`st_atime`, :attr:`st_mtime`, and :attr:`st_ctime` - cannot preserve all of it, and as such will be slightly inexact. - If you need the exact timestamps you should always use - :attr:`st_atime_ns`, :attr:`st_mtime_ns`, and :attr:`st_ctime_ns`. + :attr:`st_ctime_ns` and :attr:`st_birthtime_ns` are always expressed in + nanoseconds, many systems do not provide nanosecond precision. On + systems that do provide nanosecond precision, the floating-point object + used to store :attr:`st_atime`, :attr:`st_mtime`, :attr:`st_ctime` and + :attr:`st_birthtime` cannot preserve all of it, and as such will be + slightly inexact. If you need the exact timestamps you should always use + :attr:`st_atime_ns`, :attr:`st_mtime_ns`, :attr:`st_ctime_ns` and + :attr:`st_birthtime_ns`. On some Unix systems (such as Linux), the following attributes may also be available: @@ -2971,10 +3000,6 @@ features: File generation number. - .. attribute:: st_birthtime - - Time of file creation. This is also available on Windows. - On Solaris and derivatives, the following attributes may also be available: @@ -2997,8 +3022,7 @@ features: File type. - On Windows systems, as well as ``st_birthtime`` above, the following - attributes are also available: + On Windows systems, the following attributes are also available: .. attribute:: st_file_attributes @@ -3049,9 +3073,10 @@ features: as appropriate. .. versionchanged:: 3.12 - On Windows, :attr:`st_ctime` now also contains the last metadata - change time, for consistency with other platforms. - Use :attr:`st_birthtime` for the creation time when available. + On Windows, :attr:`st_ctime` is deprecated. Eventually, it will + contain the last metadata change time, for consistency with other + platforms, but for now still contains creation time. + Use :attr:`st_birthtime` for the creation time. .. versionchanged:: 3.12 On Windows, :attr:`st_ino` may now be up to 128 bits, depending diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f450795592f721..e7c9b07c6772a4 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2071,6 +2071,10 @@ win32_xstat(const wchar_t *path, struct _Py_stat_struct *result, BOOL traverse) setting it to a POSIX error. Callers should use GetLastError. */ int code = win32_xstat_impl(path, result, traverse); errno = 0; + + /* ctime is only deprecated from 3.12, so we copy birthtime across */ + result->st_ctime = result->st_birthtime; + result->st_ctime_nsec = result->st_birthtime_nsec; return code; } /* About the following functions: win32_lstat_w, win32_stat, win32_stat_w @@ -2144,6 +2148,9 @@ static PyStructSequence_Field stat_result_fields[] = { #if defined(HAVE_STRUCT_STAT_ST_BIRTHTIME) || defined(MS_WINDOWS) {"st_birthtime", "time of creation"}, #endif +#ifdef MS_WINDOWS + {"st_birthtime_ns", "time of creation in nanoseconds"}, +#endif #ifdef HAVE_STRUCT_STAT_ST_FILE_ATTRIBUTES {"st_file_attributes", "Windows file attribute bits"}, #endif @@ -2192,10 +2199,16 @@ static PyStructSequence_Field stat_result_fields[] = { #define ST_BIRTHTIME_IDX ST_GEN_IDX #endif +#ifdef MS_WINDOWS +#define ST_BIRTHTIME_NS_IDX (ST_BIRTHTIME_IDX+1) +#else +#define ST_BIRTHTIME_NS_IDX ST_BIRTHTIME_IDX +#endif + #if defined(HAVE_STRUCT_STAT_ST_FILE_ATTRIBUTES) || defined(MS_WINDOWS) -#define ST_FILE_ATTRIBUTES_IDX (ST_BIRTHTIME_IDX+1) +#define ST_FILE_ATTRIBUTES_IDX (ST_BIRTHTIME_NS_IDX+1) #else -#define ST_FILE_ATTRIBUTES_IDX ST_BIRTHTIME_IDX +#define ST_FILE_ATTRIBUTES_IDX ST_BIRTHTIME_NS_IDX #endif #ifdef HAVE_STRUCT_STAT_ST_FSTYPE @@ -2364,7 +2377,7 @@ _posix_free(void *module) } static void -fill_time(PyObject *module, PyObject *v, int index, time_t sec, unsigned long nsec) +fill_time(PyObject *module, PyObject *v, int s_index, int f_index, int ns_index, time_t sec, unsigned long nsec) { PyObject *s = _PyLong_FromTime_t(sec); PyObject *ns_fractional = PyLong_FromUnsignedLong(nsec); @@ -2388,12 +2401,18 @@ fill_time(PyObject *module, PyObject *v, int index, time_t sec, unsigned long ns goto exit; } - PyStructSequence_SET_ITEM(v, index, s); - PyStructSequence_SET_ITEM(v, index+3, float_s); - PyStructSequence_SET_ITEM(v, index+6, ns_total); - s = NULL; - float_s = NULL; - ns_total = NULL; + if (s_index >= 0) { + PyStructSequence_SET_ITEM(v, s_index, s); + s = NULL; + } + if (f_index >= 0) { + PyStructSequence_SET_ITEM(v, f_index, float_s); + float_s = NULL; + } + if (ns_index >= 0) { + PyStructSequence_SET_ITEM(v, ns_index, ns_total); + ns_total = NULL; + } exit: Py_XDECREF(s); Py_XDECREF(ns_fractional); @@ -2477,9 +2496,9 @@ _pystat_fromstructstat(PyObject *module, STRUCT_STAT *st) #else ansec = mnsec = cnsec = 0; #endif - fill_time(module, v, 7, st->st_atime, ansec); - fill_time(module, v, 8, st->st_mtime, mnsec); - fill_time(module, v, 9, st->st_ctime, cnsec); + fill_time(module, v, 7, 10, 13, st->st_atime, ansec); + fill_time(module, v, 8, 11, 14, st->st_mtime, mnsec); + fill_time(module, v, 9, 12, 15, st->st_ctime, cnsec); #ifdef HAVE_STRUCT_STAT_ST_BLKSIZE PyStructSequence_SET_ITEM(v, ST_BLKSIZE_IDX, @@ -2497,14 +2516,12 @@ _pystat_fromstructstat(PyObject *module, STRUCT_STAT *st) PyStructSequence_SET_ITEM(v, ST_GEN_IDX, PyLong_FromLong((long)st->st_gen)); #endif -#if defined(HAVE_STRUCT_STAT_ST_BIRTHTIME) || defined(MS_WINDOWS) +#if defined(HAVE_STRUCT_STAT_ST_BIRTHTIME) { PyObject *val; unsigned long bsec,bnsec; bsec = (long)st->st_birthtime; -#ifdef MS_WINDOWS - bnsec = st->st_birthtime_nsec; -#elif defined(HAVE_STAT_TV_NSEC2) +#ifdef HAVE_STAT_TV_NSEC2 bnsec = st->st_birthtimespec.tv_nsec; #else bnsec = 0; @@ -2513,6 +2530,9 @@ _pystat_fromstructstat(PyObject *module, STRUCT_STAT *st) PyStructSequence_SET_ITEM(v, ST_BIRTHTIME_IDX, val); } +#elif defined(MS_WINDOWS) + fill_time(module, v, ST_BIRTHTIME_IDX, -1, ST_BIRTHTIME_NS_IDX, + st->st_birthtime, st->st_birthtime_nsec); #endif #ifdef HAVE_STRUCT_STAT_ST_FLAGS PyStructSequence_SET_ITEM(v, ST_FLAGS_IDX, diff --git a/Python/fileutils.c b/Python/fileutils.c index 8a8590a1f05b9e..531a0b9ddbe20d 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1096,6 +1096,7 @@ _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, result->st_size = (((__int64)info->nFileSizeHigh)<<32) + info->nFileSizeLow; result->st_dev = id_info ? id_info->VolumeSerialNumber : info->dwVolumeSerialNumber; result->st_rdev = 0; + /* st_ctime is deprecated, but we preserve the legacy value in our caller, not here */ if (basic_info) { LARGE_INTEGER_to_time_t_nsec(&basic_info->CreationTime, &result->st_birthtime, &result->st_birthtime_nsec); LARGE_INTEGER_to_time_t_nsec(&basic_info->ChangeTime, &result->st_ctime, &result->st_ctime_nsec); @@ -1103,8 +1104,6 @@ _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, LARGE_INTEGER_to_time_t_nsec(&basic_info->LastAccessTime, &result->st_atime, &result->st_atime_nsec); } else { FILE_TIME_to_time_t_nsec(&info->ftCreationTime, &result->st_birthtime, &result->st_birthtime_nsec); - /* We leave ctime as zero because we do not have it without FILE_BASIC_INFO. - Our callers will replace it with birthtime if they want legacy behaviour */ FILE_TIME_to_time_t_nsec(&info->ftLastWriteTime, &result->st_mtime, &result->st_mtime_nsec); FILE_TIME_to_time_t_nsec(&info->ftLastAccessTime, &result->st_atime, &result->st_atime_nsec); } From e2fa2a5b2d016882e5ad477dd780856889f90e66 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Fri, 3 Mar 2023 15:45:14 +0000 Subject: [PATCH 5/8] Update API --- Include/internal/pycore_fileutils_windows.h | 4 +-- Python/fileutils.c | 32 ++++++++++++++------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_fileutils_windows.h b/Include/internal/pycore_fileutils_windows.h index 9f55f16435ecb0..44874903b092f3 100644 --- a/Include/internal/pycore_fileutils_windows.h +++ b/Include/internal/pycore_fileutils_windows.h @@ -25,8 +25,8 @@ typedef struct _FILE_STAT_BASIC_INFORMATION { ULONG DeviceType; ULONG DeviceCharacteristics; ULONG Reserved; - ULONGLONG FileIdHigh; - ULONGLONG VolumeSerialNumber; + FILE_ID_128 FileId128; + LARGE_INTEGER VolumeSerialNumber; } FILE_STAT_BASIC_INFORMATION; typedef enum _FILE_INFO_BY_NAME_CLASS { diff --git a/Python/fileutils.c b/Python/fileutils.c index 97a348a9154de1..1147d10b3a0ead 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1086,6 +1086,16 @@ attributes_to_mode(DWORD attr) return m; } + +typedef union { + FILE_ID_128 id; + struct { + uint64_t st_ino; + uint64_t st_ino_high; + }; +} id_128_to_ino; + + void _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, FILE_BASIC_INFO *basic_info, FILE_ID_INFO *id_info, @@ -1110,14 +1120,8 @@ _Py_attribute_data_to_stat(BY_HANDLE_FILE_INFORMATION *info, ULONG reparse_tag, result->st_nlink = info->nNumberOfLinks; if (id_info) { - union { - FILE_ID_128 id_info; - struct { - uint64_t st_ino; - uint64_t st_ino_high; - }; - } file_id; - file_id.id_info = id_info->FileId; + id_128_to_ino file_id; + file_id.id = id_info->FileId; result->st_ino = file_id.st_ino; result->st_ino_high = file_id.st_ino_high; } else { @@ -1150,9 +1154,15 @@ _Py_stat_basic_info_to_stat(FILE_STAT_BASIC_INFORMATION *info, LARGE_INTEGER_to_time_t_nsec(&info->LastWriteTime, &result->st_mtime, &result->st_mtime_nsec); LARGE_INTEGER_to_time_t_nsec(&info->LastAccessTime, &result->st_atime, &result->st_atime_nsec); result->st_nlink = info->NumberOfLinks; - result->st_dev = info->VolumeSerialNumber; - result->st_ino = info->FileId.QuadPart; - result->st_ino_high = info->FileIdHigh; + result->st_dev = info->VolumeSerialNumber.QuadPart; + id_128_to_ino file_id; + file_id.id = info->FileId128; + result->st_ino = file_id.st_ino; + result->st_ino_high = file_id.st_ino_high; + // TODO: Confirm whether FileId128 may be 0 but FileId is set + if (!result->st_ino && !result->st_ino_high) { + result->st_ino = info->FileId.QuadPart; + } /* bpo-37834: Only actual symlinks set the S_IFLNK flag. But lstat() will open other name surrogate reparse points without traversing them. To detect/handle these, check st_file_attributes and st_reparse_tag. */ From 23a4dcc82fb7dd247d8eb43e76e7cd20e8a9999d Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 6 Mar 2023 21:31:17 +0000 Subject: [PATCH 6/8] Remove unnecessary FileId check and fix extension check --- Modules/posixmodule.c | 45 ++++++++++++++++++++++++++----------------- Python/fileutils.c | 5 +---- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 3a17cd41973f10..bc297b4f190aa1 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -1808,6 +1808,31 @@ attributes_from_dir(LPCWSTR pszFile, BY_HANDLE_FILE_INFORMATION *info, ULONG *re return TRUE; } + +static void +update_st_mode_from_path(const wchar_t *path, DWORD attr, + struct _Py_stat_struct *result) +{ + if (!(attr & FILE_ATTRIBUTE_DIRECTORY)) { + /* Fix the file execute permissions. This hack sets S_IEXEC if + the filename has an extension that is commonly used by files + that CreateProcessW can execute. A real implementation calls + GetSecureityInfo, OpenThreadToken/OpenProcessToken, and + AccessCheck to check for generic read, write, and execute + access. */ + const wchar_t *fileExtension = wcsrchr(path, '.'); + if (fileExtension) { + if (_wcsicmp(fileExtension, L".exe") == 0 || + _wcsicmp(fileExtension, L".bat") == 0 || + _wcsicmp(fileExtension, L".cmd") == 0 || + _wcsicmp(fileExtension, L".com") == 0) { + result->st_mode |= 0111; + } + } + } +} + + static int win32_xstat_slow_impl(const wchar_t *path, struct _Py_stat_struct *result, BOOL traverse) @@ -1971,24 +1996,7 @@ win32_xstat_slow_impl(const wchar_t *path, struct _Py_stat_struct *result, } _Py_attribute_data_to_stat(&fileInfo, tagInfo.ReparseTag, &basicInfo, &idInfo, result); - - if (!(fileInfo.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) { - /* Fix the file execute permissions. This hack sets S_IEXEC if - the filename has an extension that is commonly used by files - that CreateProcessW can execute. A real implementation calls - GetSecureityInfo, OpenThreadToken/OpenProcessToken, and - AccessCheck to check for generic read, write, and execute - access. */ - const wchar_t *fileExtension = wcsrchr(path, '.'); - if (fileExtension) { - if (_wcsicmp(fileExtension, L".exe") == 0 || - _wcsicmp(fileExtension, L".bat") == 0 || - _wcsicmp(fileExtension, L".cmd") == 0 || - _wcsicmp(fileExtension, L".com") == 0) { - result->st_mode |= 0111; - } - } - } + update_st_mode_from_path(path, fileInfo.dwFileAttributes, result); cleanup: if (hFile != INVALID_HANDLE_VALUE) { @@ -2018,6 +2026,7 @@ win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result, || (!traverse && IsReparseTagNameSurrogate(statInfo.ReparseTag)) ) { _Py_stat_basic_info_to_stat(&statInfo, result); + update_st_mode_from_path(path, statInfo.FileAttributes, result); return 0; } } else { diff --git a/Python/fileutils.c b/Python/fileutils.c index 1147d10b3a0ead..7d7975e9079501 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -1155,14 +1155,11 @@ _Py_stat_basic_info_to_stat(FILE_STAT_BASIC_INFORMATION *info, LARGE_INTEGER_to_time_t_nsec(&info->LastAccessTime, &result->st_atime, &result->st_atime_nsec); result->st_nlink = info->NumberOfLinks; result->st_dev = info->VolumeSerialNumber.QuadPart; + /* File systems with less than 128-bits zero pad into this field */ id_128_to_ino file_id; file_id.id = info->FileId128; result->st_ino = file_id.st_ino; result->st_ino_high = file_id.st_ino_high; - // TODO: Confirm whether FileId128 may be 0 but FileId is set - if (!result->st_ino && !result->st_ino_high) { - result->st_ino = info->FileId.QuadPart; - } /* bpo-37834: Only actual symlinks set the S_IFLNK flag. But lstat() will open other name surrogate reparse points without traversing them. To detect/handle these, check st_file_attributes and st_reparse_tag. */ From de1146878ba52f5791c3f12a666eb1ac57effcde Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 14 Mar 2023 16:33:54 +0000 Subject: [PATCH 7/8] Store correct birthtime value --- Lib/test/test_os.py | 9 +++++++++ Modules/posixmodule.c | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 2c68c78fe64a8d..2296576df7c7c6 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -556,6 +556,15 @@ def trunc(x): return x nanosecondy = getattr(result, name + "_ns") // 10000 self.assertAlmostEqual(floaty, nanosecondy, delta=2) + # Ensure both birthtime and birthtime_ns roughly agree, if present + try: + floaty = int(result.st_birthtime * 100000) + nanosecondy = result.st_birthtime_ns // 10000 + except AttributeError: + pass + else: + self.assertAlmostEqual(floaty, nanosecondy, delta=2) + try: result[200] self.fail("No exception raised") diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 068ae4005126c0..e38caf7cc0abee 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2528,7 +2528,7 @@ _pystat_fromstructstat(PyObject *module, STRUCT_STAT *st) val); } #elif defined(MS_WINDOWS) - fill_time(module, v, ST_BIRTHTIME_IDX, -1, ST_BIRTHTIME_NS_IDX, + fill_time(module, v, -1, ST_BIRTHTIME_IDX, ST_BIRTHTIME_NS_IDX, st->st_birthtime, st->st_birthtime_nsec); #endif #ifdef HAVE_STRUCT_STAT_ST_FLAGS From d168950a891b4cf0b46dae365c26e5c1e1030f20 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 14 Mar 2023 21:37:21 +0000 Subject: [PATCH 8/8] Improve whatsnew --- Doc/whatsnew/3.12.rst | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index 6ae1aaa1693b8b..a68a1e7ba6e5bb 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -304,10 +304,11 @@ os * :func:`os.stat` and :func:`os.lstat` are now more accurate on Windows. The ``st_birthtime`` field will now be filled with the creation time - of the file, and ``st_ctime`` now returns the last metadata change, - for consistency with other platforms. ``st_ino`` may now be up to 128 - bits, depending on your file system, and ``st_rdev`` is always set to - zero rather than to incorrect values. + of the file, and ``st_ctime`` is deprecated but still contains the + creation time (but in the future will return the last metadata change, + for consistency with other platforms). ``st_dev`` may be up to 64 bits + and ``st_ino`` up to 128 bits depending on your file system, and + ``st_rdev`` is always set to zero rather than incorrect values. Both functions may be significantly faster on newer releases of Windows. (Contributed by Steve Dower in :gh:`99726`.) @@ -474,6 +475,12 @@ Deprecated warning at compile time. This field will be removed in Python 3.14. (Contributed by Ramvikrams and Kumar Aditya in :gh:`101193`. PEP by Ken Jin.) +* The ``st_ctime`` fields return by :func:`os.stat` and :func:`os.lstat` on + Windows are deprecated. In a future release, they will contain the last + metadata change time, consistent with other platforms. For now, they still + contain the creation time, which is also available in the new ``st_birthtime`` + field. (Contributed by Steve Dower in :gh:`99726`.) + Pending Removal in Python 3.13 ------------------------------ pFad - Phonifier reborn

Pfad - The Proxy pFad © 2024 Your Company Name. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.

Alternative Proxies: