From e6bd6cb462cf3bfc0ca7ee53cc36b5fca0704fe9 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Mon, 27 Jul 2026 10:34:46 +0100 Subject: [PATCH 1/7] Add a 'cleared` argument to `import_fresh_module` to allow modules to be removed without forcibly re-importing them. --- Lib/test/support/import_helper.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index e8a3d176ad6943f..2625720e6b5ae17 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -139,7 +139,7 @@ def multi_interp_extensions_check(enabled=True): _imp._override_multi_interp_extensions_check(old) -def import_fresh_module(name, fresh=(), blocked=(), *, +def import_fresh_module(name, fresh=(), blocked=(), cleared=(), *, deprecated=False, usefrozen=False, ): @@ -158,7 +158,12 @@ def import_fresh_module(name, fresh=(), blocked=(), *, in the module cache during the import to ensure that attempts to import them raise ImportError. - The named module and any modules named in the *fresh* and *blocked* + *cleared* is an iterable of module names that are removed from the + sys.modules cache before doing the import but *are not* re-imported. + This is to allow for the case where these module imports may fail as part + of the test due to other modules that are blocked. + + The named module and any modules named in the *fresh*, *blocked* and *cleared* parameters are saved before starting the import and then reinserted into sys.modules when the fresh import is complete. @@ -178,7 +183,8 @@ def import_fresh_module(name, fresh=(), blocked=(), *, # as those which just need a blocking entry removed fresh = list(fresh) blocked = list(blocked) - names = {name, *fresh, *blocked} + cleared = list(cleared) + names = {name, *fresh, *blocked, *cleared} orig_modules = _save_and_remove_modules(names) for modname in blocked: sys.modules[modname] = None From a7dc663c48870f479df3e3db750e154f2aa7bc9d Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Mon, 27 Jul 2026 10:36:09 +0100 Subject: [PATCH 2/7] Document the new `cleared` argument to `import_fresh_module`, add the undocumented `usefrozen` argument and correct the signature as `deprecated` and `usefrozen` are keyword only arguments. --- Doc/library/test.rst | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 4e21e1ded82724c..00d46fc670a2fce 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1607,7 +1607,7 @@ The :mod:`!test.support.import_helper` module provides support for import tests. byte-compiled files of the module. -.. function:: import_fresh_module(name, fresh=(), blocked=(), deprecated=False) +.. function:: import_fresh_module(name, fresh=(), blocked=(), cleared=(), *, deprecated=False, usefrozen=False) This function imports and returns a fresh copy of the named Python module by removing the named module from ``sys.modules`` before doing the import. @@ -1621,6 +1621,11 @@ The :mod:`!test.support.import_helper` module provides support for import tests. in the module cache during the import to ensure that attempts to import them raise :exc:`ImportError`. + *cleared* is an iterable of module names that are removed from the + sys.modules cache before doing the import but *are not* re-imported. + This is to allow for the case where these module imports may fail as part + of the test due to other modules that are blocked. + The named module and any modules named in the *fresh* and *blocked* parameters are saved before starting the import and then reinserted into ``sys.modules`` when the fresh import is complete. @@ -1631,6 +1636,9 @@ The :mod:`!test.support.import_helper` module provides support for import tests. This function will raise :exc:`ImportError` if the named module cannot be imported. + If "usefrozen" is False (the default) then the frozen importer is + disabled (except for essential modules like importlib._bootstrap). + Example use:: # Get copies of the warnings module for testing without affecting the From a78fbfa1041b14a9d262ad267c77109b13dea2f5 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Mon, 27 Jul 2026 10:36:25 +0100 Subject: [PATCH 3/7] Add a test for format support under lazy imports --- Lib/test/test_shutil.py | 61 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 6832bea094fc1dc..e6aa1fcd14349fc 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -32,6 +32,7 @@ from test import support from test.support import os_helper, socket_helper from test.support.os_helper import TESTFN, FakePath +from test.support.import_helper import import_fresh_module TESTFN2 = TESTFN + "2" TESTFN_SRC = TESTFN + "_SRC" @@ -2322,6 +2323,66 @@ def _boo(filename, extract_dir, extra): self.assertEqual(get_unpack_formats(), formats) +@support.requires_bz2() +@support.requires_lzma() +@support.requires_zlib() +@support.requires_zstd() +class TestLazyArchiveDetection(unittest.TestCase): + # Test archive detection under 'all' lazy imports + # see: https://github.com/python/cpython/issues/150167 + @support.thread_unsafe("Modifies global import state") + def test_lazy_success(self): + import_state = sys.get_lazy_imports() + try: + sys.set_lazy_imports("all") + lazy_shutil = import_fresh_module( + "shutil", + cleared=("bz2", "lzma", "zlib", "compression", "compression.zstd"), + ) + finally: + sys.set_lazy_imports(import_state) + + assert lazy_shutil is not None # type narrowing + + formats = { + "bz2": lazy_shutil._BZ2_SUPPORTED, + "lzma": lazy_shutil._LZMA_SUPPORTED, + "zlib": lazy_shutil._ZLIB_SUPPORTED, + "zstd": lazy_shutil._ZSTD_SUPPORTED, + } + + expected = {k: True for k in formats} + + self.assertEqual(formats, expected) + + + @support.thread_unsafe("Modifies global import state") + def test_lazy_failure(self): + import_state = sys.get_lazy_imports() + try: + sys.set_lazy_imports("all") + lazy_shutil = import_fresh_module( + "shutil", + blocked=("_bz2", "_lzma", "zlib", "_zstd"), + cleared=("bz2", "lzma", "compression", "compression.zstd"), + ) + finally: + sys.set_lazy_imports(import_state) + + assert lazy_shutil is not None # type narrowing + + formats = { + "bz2": lazy_shutil._BZ2_SUPPORTED, + "lzma": lazy_shutil._LZMA_SUPPORTED, + "zlib": lazy_shutil._ZLIB_SUPPORTED, + "zstd": lazy_shutil._ZSTD_SUPPORTED, + } + + expected = {k: False for k in formats} + + self.assertEqual(formats, expected) + + class TestMisc(BaseTest, unittest.TestCase): @unittest.skipUnless(hasattr(shutil, 'disk_usage'), From b94198b7ec6fd886657346775d9422a5b565b799 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Mon, 27 Jul 2026 11:47:08 +0100 Subject: [PATCH 4/7] Document that import_fresh_module has changed. --- Doc/library/test.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 00d46fc670a2fce..c403384fb47362f 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1637,7 +1637,7 @@ The :mod:`!test.support.import_helper` module provides support for import tests. imported. If "usefrozen" is False (the default) then the frozen importer is - disabled (except for essential modules like importlib._bootstrap). + disabled (except for essential modules like ``importlib._bootstrap``). Example use:: @@ -1649,6 +1649,8 @@ The :mod:`!test.support.import_helper` module provides support for import tests. c_warnings = import_fresh_module('warnings', fresh=['_warnings']) .. versionadded:: 3.1 + .. versionchanged:: next + New optional argument *cleared*. .. function:: import_module(name, deprecated=False, *, required_on=()) From a064e28685a3f090b3cdfc89778666fec61f07f3 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Mon, 27 Jul 2026 12:15:38 +0100 Subject: [PATCH 5/7] Force the eager import of `_bz2` inside `bz2.py` --- Lib/bz2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/bz2.py b/Lib/bz2.py index eb58f4da596ea18..418f67c248047b4 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -14,6 +14,7 @@ import io import os +import _bz2; _bz2 # Force the eager import of _bz2 from _bz2 import BZ2Compressor, BZ2Decompressor From 638ce31c3b5d26786c1f774883cd099cbb9de81c Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Mon, 27 Jul 2026 13:11:28 +0100 Subject: [PATCH 6/7] Add a link to the issue explaining why `_bz2` is being eagerly imported --- Lib/bz2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/bz2.py b/Lib/bz2.py index 418f67c248047b4..a682f39c64dbff9 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -14,7 +14,9 @@ import io import os -import _bz2; _bz2 # Force the eager import of _bz2 +# Force the eager import of _bz2 +# See: https://github.com/python/cpython/issues/150167 +import _bz2; _bz2 from _bz2 import BZ2Compressor, BZ2Decompressor From 26e9b6436b693ea872c8d5f1db6c88052d99b434 Mon Sep 17 00:00:00 2001 From: David C Ellis Date: Mon, 27 Jul 2026 13:19:05 +0100 Subject: [PATCH 7/7] Also clear the internal modules in the success test. --- Lib/test/test_shutil.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index e6aa1fcd14349fc..baa225eb620ed49 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -2337,7 +2337,16 @@ def test_lazy_success(self): sys.set_lazy_imports("all") lazy_shutil = import_fresh_module( "shutil", - cleared=("bz2", "lzma", "zlib", "compression", "compression.zstd"), + cleared=( + "bz2", + "lzma", + "zlib", + "compression", + "compression.zstd", + "_bz2", + "_lzma", + "_zstd", + ), ) finally: sys.set_lazy_imports(import_state)