diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst index b7c6550ff34aac..487962800bc6dc 100644 --- a/Doc/c-api/contextvars.rst +++ b/Doc/c-api/contextvars.rst @@ -157,6 +157,14 @@ Context variable functions: a default value for the context variable, or ``NULL`` for no default. If an error has occurred, this function returns ``NULL``. +.. c:function:: PyObject *PyContextVar_NewThreadInheritable(const char *name, PyObject *def) + + Create a new ``ContextVar`` object whose bindings are inherited by new + :class:`threading.Thread` instances. The parameters and return value are + the same as for :c:func:`PyContextVar_New`. + + .. versionadded:: 3.16 + .. c:function:: int PyContextVar_Get(PyObject *var, PyObject *default_value, PyObject **value) Get the value of a context variable. Returns ``-1`` if an error has diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst index 53bea1db191d76..51e24d13cc660f 100644 --- a/Doc/howto/free-threading-python.rst +++ b/Doc/howto/free-threading-python.rst @@ -152,8 +152,9 @@ is set to true by default which causes threads created with :class:`threading.Thread` to start with a copy of the :class:`~contextvars.Context()` of the caller of :meth:`~threading.Thread.start`. In the default GIL-enabled build, the flag -defaults to false so threads start with an -empty :class:`~contextvars.Context()`. +defaults to false, so threads start with a context containing only the caller's +current bindings for context variables created by +:meth:`~contextvars.ContextVar.thread_inheritable`. Warning filters diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index b0cc0be8e911bf..eedbca31fc11fa 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -45,6 +45,72 @@ Context Variables :class:`!ContextVar`\s are :ref:`generic ` over the type of their contained value. + .. classmethod:: ContextVar.thread_inheritable(name, [*, default]) + + Return a new context variable whose binding is inherited by new + :class:`threading.Thread` instances. When + :meth:`threading.Thread.start` would otherwise start the thread with + an empty context (that is, when + :data:`sys.flags.thread_inherit_context` is false and no explicit + :class:`Context` was supplied), the new thread's context is initialized + with the caller's current bindings for all thread-inheritable variables. + This allows individual context variables to opt in to thread inheritance + on a per-variable basis. + + These are ordinary bindings in the new thread's context: they are + visible to :func:`copy_context`, may be changed independently with + :meth:`ContextVar.set`, and are in turn inherited by threads started + from the new thread. Threads started with an explicitly supplied + context are unaffected, as are threads started by means other than + :meth:`threading.Thread.start`, such as + :func:`_thread.start_new_thread` or the C API. + + The *name* and *default* parameters have the same meaning as for the + :class:`ContextVar` constructor. When + :data:`sys.flags.thread_inherit_context` is true, a variable created + with this method behaves identically to one created with + :class:`ContextVar`, so it is safe to use unconditionally. + + Libraries that also support Python versions without this method must + choose how to degrade on those versions. For state with a safe + default in a new thread (the behavior that :class:`threading.local` + based state has always had), fall back to an ordinary context + variable:: + + _new_var = getattr(ContextVar, "thread_inheritable", ContextVar) + var = _new_var("var") + + With this fallback, on older versions new threads see the variable's + default rather than the starting thread's binding, unless the + application enables :option:`-X thread_inherit_context <-X>`. + + For state that was previously a module-level global, reverting to the + default in new threads may break existing threaded code, since such + code has always seen the current global value. Those libraries can + instead pick the best semantics each interpreter supports:: + + if hasattr(ContextVar, "thread_inheritable"): + # Context-local and inherited by new threads. + _new_var = ContextVar.thread_inheritable + elif getattr(sys.flags, "thread_inherit_context", False): + # Threads inherit the whole context, so an ordinary + # context variable is inherited too. + _new_var = ContextVar + else: + # Keep the library's historical global semantics. + _new_var = _GlobalVar + + Here ``_GlobalVar`` is a class provided by the library that stores a + single process-wide value. To be substitutable for + :class:`ContextVar` it must accept the same constructor arguments + (*name* positional, keyword-only *default*), distinguish a missing + *default* from ``default=None``, and provide ``get()``, ``set()`` + returning a token, and ``reset()``. This case gives up task-local + isolation: concurrent tasks and threads share one value, exactly as + they did before the library adopted context variables. + + .. versionadded:: 3.16 + .. attribute:: ContextVar.name The name of the variable. This is a read-only property. diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 5d9a7b6314b166..4ae75cf8d3cf8a 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -539,12 +539,24 @@ since it is impossible to detect the termination of alien threads. the thread. The default value is ``None`` which indicates that the :data:`sys.flags.thread_inherit_context` flag controls the behaviour. If the flag is true, threads will start with a copy of the context of the - caller of :meth:`~Thread.start`. If false, they will start with an empty - context. To explicitly start with an empty context, pass a new instance of + caller of :meth:`~Thread.start`. If false, they will start with a + context containing only the caller's current bindings for context variables + created by :meth:`~contextvars.ContextVar.thread_inheritable`. To explicitly + start with an empty context, pass a new instance of :class:`~contextvars.Context()`. To explicitly start with a copy of the current context, pass the value from :func:`~contextvars.copy_context`. The flag defaults true on free-threaded builds and false otherwise. + On GIL-enabled builds, when the flag has its default value of false and + *context* is ``None``, looking up a context variable that was set in the + caller of :meth:`~Thread.start` but is absent from the new thread's context + emits a :exc:`DeprecationWarning`. In a future release, the flag will + default to true on all builds, so threads will inherit context by default. + Set + :option:`-X thread_inherit_context <-X>` or + :envvar:`PYTHON_THREAD_INHERIT_CONTEXT`, or pass an explicit *context*, to + select the behavior now and suppress this warning. + If the subclass overrides the constructor, it must make sure to invoke the base class constructor (``Thread.__init__()``) before doing anything else to the thread. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 677fbbae3f4219..6b1426da110afa 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -675,11 +675,12 @@ Miscellaneous options .. versionadded:: 3.13 * :samp:`-X thread_inherit_context={0,1}` causes :class:`~threading.Thread` - to, by default, use a copy of context of the caller of - ``Thread.start()`` when starting. Otherwise, threads will start - with an empty context. If unset, the value of this option defaults - to ``1`` on free-threaded builds and to ``0`` otherwise. See also - :envvar:`PYTHON_THREAD_INHERIT_CONTEXT`. + to, by default, use a copy of the context of the caller of + ``Thread.start()`` when starting. Otherwise, threads start with a context + containing only the caller's current bindings for context variables + created by :meth:`~contextvars.ContextVar.thread_inheritable`. If unset, the + value of this option defaults to ``1`` on free-threaded builds and to ``0`` + otherwise. See also :envvar:`PYTHON_THREAD_INHERIT_CONTEXT`. .. versionadded:: 3.14 @@ -1367,10 +1368,12 @@ conflict. .. envvar:: PYTHON_THREAD_INHERIT_CONTEXT If this variable is set to ``1`` then :class:`~threading.Thread` will, - by default, use a copy of context of the caller of ``Thread.start()`` - when starting. Otherwise, new threads will start with an empty context. - If unset, this variable defaults to ``1`` on free-threaded builds and to - ``0`` otherwise. See also :option:`-X thread_inherit_context<-X>`. + by default, use a copy of the context of the caller of ``Thread.start()`` + when starting. Otherwise, new threads start with a context containing only + the caller's current bindings for context variables created by + :meth:`~contextvars.ContextVar.thread_inheritable`. If unset, this variable + defaults to ``1`` on free-threaded builds and to ``0`` otherwise. See also + :option:`-X thread_inherit_context<-X>`. .. versionadded:: 3.14 diff --git a/Include/cpython/context.h b/Include/cpython/context.h index 3a7a4b459c09ad..70c33c45e962e4 100644 --- a/Include/cpython/context.h +++ b/Include/cpython/context.h @@ -68,6 +68,13 @@ PyAPI_FUNC(int) PyContext_ClearWatcher(int watcher_id); PyAPI_FUNC(PyObject *) PyContextVar_New( const char *name, PyObject *default_value); +/* Create a new thread-inheritable context variable. + + default_value can be NULL. +*/ +PyAPI_FUNC(PyObject *) PyContextVar_NewThreadInheritable( + const char *name, PyObject *default_value); + /* Get a value for the variable. diff --git a/Include/internal/pycore_context.h b/Include/internal/pycore_context.h index a833f790a621b1..0b179332815aba 100644 --- a/Include/internal/pycore_context.h +++ b/Include/internal/pycore_context.h @@ -26,6 +26,18 @@ struct _pycontextobject { PyHamtObject *ctx_vars; PyObject *ctx_weakreflist; int ctx_entered; + // Redundant subset of ctx_vars holding only the bindings of + // thread-inheritable context variables (see + // ContextVar.thread_inheritable()). Used to efficiently create the + // starting context of a new thread. + PyHamtObject *ctx_thread_inheritable_vars; + // used to emit warnings about thread_inherit_context + PyHamtObject *ctx_starter_vars; + // Nesting depth in the context inheritance tree. Assigned at creation + // time: an empty/base context has depth 0, and a context produced by + // copying another (copy_context(), Context.copy(), thread/async context + // inheritance) has depth one greater than its source. + uint64_t ctx_depth; }; @@ -33,6 +45,7 @@ struct _pycontextvarobject { PyObject_HEAD PyObject *var_name; PyObject *var_default; + char var_thread_inheritable; #ifndef Py_GIL_DISABLED PyObject *var_cached; uint64_t var_cached_tsid; @@ -54,9 +67,14 @@ struct _pycontexttokenobject { // _testinternalcapi.hamt() used by tests. // Export for '_testcapi' shared extension PyAPI_FUNC(PyObject*) _PyContext_NewHamtForTests(void); +PyAPI_FUNC(PyObject*) _PyContext_NewForThread(void); PyAPI_FUNC(int) _PyContext_Enter(PyThreadState *ts, PyObject *octx); PyAPI_FUNC(int) _PyContext_Exit(PyThreadState *ts, PyObject *octx); +/* Return the depth (see struct _pycontextobject.ctx_depth) of the current + context, or 0 if there is no current context. */ +PyAPI_FUNC(uint64_t) _PyContext_CurrentDepth(void); + #endif /* !Py_INTERNAL_CONTEXT_H */ diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 183b2d45c5ede1..0b1cb440ece37a 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -179,6 +179,7 @@ extern PyStatus _PyConfig_SetPyArgv( PyConfig *config, const _PyArgv *args); extern PyObject* _PyConfig_CreateXOptionsDict(const PyConfig *config); +extern int _PyConfig_ThreadInheritContextWarn(const PyConfig *config); extern void _Py_DumpPathConfig(PyThreadState *tstate); diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 3d577ff717cc6e..022664cf012d8e 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -984,6 +984,9 @@ struct _is { // One bit is set for each non-NULL entry in code_watchers uint8_t active_code_watchers; uint8_t active_context_watchers; + // True if we should warn about threads not inheriting context from the + // starting thread. + bool thread_inherit_context_warn; struct _py_object_state object_state; struct _Py_unicode_state unicode; diff --git a/Lib/_py_warnings.py b/Lib/_py_warnings.py index ab09913de6812d..1873b981ec7b6b 100644 --- a/Lib/_py_warnings.py +++ b/Lib/_py_warnings.py @@ -83,7 +83,8 @@ def _filters(self): _global_context = _GlobalContext() -_warnings_context = _contextvars.ContextVar('warnings_context') +_warnings_context = _contextvars.ContextVar.thread_inheritable( + 'warnings_context') def _get_context(): diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 8c0afd14d616e8..8b1568f9200b99 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -348,8 +348,9 @@ class FloatOperation(DecimalException, TypeError): # current context. import contextvars +from _contextvars import _current_context_depth -_current_context_var = contextvars.ContextVar('decimal_context') +_current_context_var = contextvars.ContextVar.thread_inheritable('decimal_context') _context_attributes = frozenset( ['prec', 'Emin', 'Emax', 'capitals', 'clamp', 'rounding', 'flags', 'traps'] @@ -362,18 +363,32 @@ def getcontext(): a new context and sets this thread's context. New contexts are copies of DefaultContext. """ + cur_depth = _current_context_depth() try: - return _current_context_var.get() + context = _current_context_var.get() except LookupError: context = Context() + context._local_depth = cur_depth _current_context_var.set(context) return context + if context._local_depth != cur_depth: + # The context value was inherited from another task/thread. Because + # the Context() instance is mutable, copy it to ensure that if it is + # changed, those changes are isolated from other tasks/threads. + context = context.copy() + context._local_depth = cur_depth + _current_context_var.set(context) + return context + def setcontext(context): """Set this thread's context to context.""" if context in (DefaultContext, BasicContext, ExtendedContext): context = context.copy() context.clear_flags() + # Mark the context as owned by the current context scope, so a following + # getcontext() returns this very object rather than a copy. + context._local_depth = _current_context_depth() _current_context_var.set(context) del contextvars # Don't contaminate the namespace @@ -3869,6 +3884,10 @@ class Context(object): clamp - If 1, change exponents if too high (Default 0) """ + # Depth of the contextvars context this object was bound into by + # getcontext()/setcontext(). + _local_depth = 0 + def __init__(self, prec=None, rounding=None, Emin=None, Emax=None, capitals=None, clamp=None, flags=None, traps=None, _ignored_flags=None): @@ -3951,6 +3970,8 @@ def __setattr__(self, name, value): return self._set_signal_dict(name, value) elif name == '_ignored_flags': return object.__setattr__(self, name, value) + elif name == '_local_depth': + return object.__setattr__(self, name, value) else: raise AttributeError( "'decimal.Context' object has no attribute '%s'" % name) diff --git a/Lib/test/test_context.py b/Lib/test/test_context.py index ef20495dcc01ea..8fd094e8d19682 100644 --- a/Lib/test/test_context.py +++ b/Lib/test/test_context.py @@ -4,12 +4,14 @@ import contextvars import functools import gc +import os import random import time import unittest +import warnings import weakref from test import support -from test.support import threading_helper +from test.support import script_helper, threading_helper try: from _testinternalcapi import hamt @@ -17,6 +19,16 @@ hamt = None +THREAD_INHERIT_CONTEXT_WARNING = ( + not support.Py_GIL_DISABLED + and not sys.flags.thread_inherit_context + and "thread_inherit_context" not in sys._xoptions + and (sys.flags.ignore_environment + # An empty value is treated as unset, as in Python/initconfig.c. + or not os.environ.get("PYTHON_THREAD_INHERIT_CONTEXT")) +) + + def isolated_context(func): """Needed to make reftracking test mode work.""" @functools.wraps(func) @@ -40,8 +52,47 @@ def test_context_var_new_1(self): with self.assertRaises(AttributeError): c.name = 'bbb' + inheritable = contextvars.ContextVar.thread_inheritable('inheritable') + self.assertIs(type(inheritable), contextvars.ContextVar) + self.assertEqual(inheritable.name, 'inheritable') + + inheritable_with_default = ( + contextvars.ContextVar.thread_inheritable( + 'inheritable_with_default', default=42, + ) + ) + self.assertEqual(inheritable_with_default.get(), 42) + + with self.assertRaisesRegex(TypeError, 'must be a str'): + contextvars.ContextVar.thread_inheritable(1) + with self.assertRaises(TypeError): + contextvars.ContextVar.thread_inheritable('var', None) + with self.assertRaises(TypeError): + contextvars.ContextVar('var', inherit=True) + self.assertNotEqual(hash(c), hash('aaa')) + def test_thread_inheritable_context_var_gc(self): + class Value: + pass + + def make_cycle(): + var = contextvars.ContextVar.thread_inheritable('var') + ctx = contextvars.Context() + value = Value() + value_ref = weakref.ref(value) + + def bind(): + var.set(value) + value.context = contextvars.copy_context() + + ctx.run(bind) + return value_ref + + value_ref = make_cycle() + support.gc_collect() + self.assertIsNone(value_ref()) + @isolated_context def test_context_var_repr_1(self): c = contextvars.ContextVar('a') @@ -400,12 +451,12 @@ def test_context_thread_inherit(self): cvar = contextvars.ContextVar('cvar') + results = [] + def run_context_none(): - if sys.flags.thread_inherit_context: - expected = 1 - else: - expected = None - self.assertEqual(cvar.get(None), expected) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", DeprecationWarning) + results.append((cvar.get(None), len(caught))) # By default, context is inherited based on the # sys.flags.thread_inherit_context option. @@ -420,6 +471,12 @@ def run_context_none(): thread.start() thread.join() + if sys.flags.thread_inherit_context: + self.assertEqual(results, [(1, 0), (1, 0)]) + else: + warning_count = int(THREAD_INHERIT_CONTEXT_WARNING) + self.assertEqual(results, [(None, warning_count)] * 2) + # An explicit Context value can also be passed custom_ctx = contextvars.Context() custom_var = None @@ -447,6 +504,220 @@ def run_empty(): thread.start() thread.join() + @isolated_context + @threading_helper.requires_working_threading() + @unittest.skipUnless(THREAD_INHERIT_CONTEXT_WARNING, + "requires the warning-enabled GIL-build default") + def test_context_thread_inherit_warning(self): + import threading + + call_default = contextvars.ContextVar("call_default") + var_default = contextvars.ContextVar("var_default", default="variable") + missing = contextvars.ContextVar("missing") + other = contextvars.ContextVar("other") + for var in (call_default, var_default, missing, other): + var.set("starter") + + results = [] + + def target(): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", DeprecationWarning) + # Explicit nested contexts and copies of the child context do + # not retain the compatibility metadata. + values = [ + contextvars.Context().run(call_default.get, "nested"), + contextvars.copy_context().run(call_default.get, "copy"), + call_default.get("argument"), + var_default.get(), + ] + try: + missing.get() + except LookupError: + values.append("LookupError") + # The snapshot is detached after the first mismatch, including + # for repeated lookups and other variables. + values.extend((call_default.get("again"), other.get(None))) + results.append((values, caught)) + + thread = threading.Thread(target=target) + thread.start() + thread.join() + + values, caught = results[0] + self.assertEqual(values, [ + "nested", "copy", "argument", "variable", "LookupError", + "again", None, + ]) + self.assertEqual(len(caught), 1) + self.assertIs(caught[0].category, DeprecationWarning) + message = str(caught[0].message) + self.assertIn( + "threads will inherit context by default", message) + + @isolated_context + @threading_helper.requires_working_threading() + @unittest.skipIf(sys.flags.thread_inherit_context, + "requires the non-inheriting thread default") + def test_context_thread_inherit_warning_not_emitted(self): + import threading + + unbound = contextvars.ContextVar("unbound") + bound = contextvars.ContextVar("bound") + bound.set("starter") + results = [] + + def no_starter_binding(): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", DeprecationWarning) + results.append((unbound.get(None), len(caught))) + + def child_binding(): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", DeprecationWarning) + bound.set("child") + results.append((bound.get(), len(caught))) + + def explicit_empty(): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", DeprecationWarning) + results.append((bound.get(None), len(caught))) + + threads = [ + threading.Thread(target=no_starter_binding), + threading.Thread(target=child_binding), + threading.Thread(target=explicit_empty, + context=contextvars.Context()), + ] + for thread in threads: + thread.start() + thread.join() + + self.assertEqual(results, [(None, 0), ("child", 0), (None, 0)]) + + @threading_helper.requires_working_threading() + def test_context_thread_inherit_warning_explicitly_disabled(self): + code = """ +import contextvars +import threading +import warnings + +var = contextvars.ContextVar('var') +var.set('starter') +result = [] + +def target(): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter('always', DeprecationWarning) + result.append((var.get(None), len(caught))) + +thread = threading.Thread(target=target) +thread.start() +thread.join() +assert result == [(None, 0)], result +""" + script_helper.assert_python_ok( + "-X", "thread_inherit_context=0", "-c", code) + script_helper.assert_python_ok( + "-c", code, PYTHON_THREAD_INHERIT_CONTEXT="0") + + @isolated_context + @threading_helper.requires_working_threading() + @unittest.skipIf(sys.flags.thread_inherit_context, + "requires the non-inheriting thread default") + def test_context_thread_warning_snapshot_at_start(self): + import threading + + var = contextvars.ContextVar("var") + # Keep another variable bound so the starter context is non-empty at + # start() and a snapshot is retained. + keep = contextvars.ContextVar("keep") + keep.set("starter") + token = var.set("during construction") + results = [] + + def target(): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always", DeprecationWarning) + results.append((var.get(None), len(caught))) + + thread = threading.Thread(target=target) + var.reset(token) + thread.start() + thread.join() + + self.assertEqual(results, [(None, 0)]) + + @isolated_context + @threading_helper.requires_working_threading() + @unittest.skipUnless(THREAD_INHERIT_CONTEXT_WARNING, + "requires the warning-enabled GIL-build default") + def test_context_thread_inherit_warning_as_error(self): + import threading + + var = contextvars.ContextVar("var") + var.set("starter") + results = [] + + def target(): + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + try: + var.get() + except DeprecationWarning as exc: + results.append(str(exc)) + + thread = threading.Thread(target=target) + thread.start() + thread.join() + + self.assertEqual(len(results), 1) + self.assertIn( + "threads will inherit context by default", + results[0]) + + @threading_helper.requires_working_threading() + @unittest.skipIf(support.Py_GIL_DISABLED, + "the free-threaded default inherits context") + def test_context_thread_inherit_warning_context_aware(self): + code = """ +import contextvars +import threading +import warnings + +var = contextvars.ContextVar('var') +var.set('starter') +result = [] + +def target(): + try: + var.get() + except DeprecationWarning: + result.append('warning') + +with warnings.catch_warnings(): + warnings.simplefilter('error', DeprecationWarning) + thread = threading.Thread(target=target) + thread.start() + thread.join() + +assert result == ['warning'], result +""" + # An empty value is treated as unset; this shields the subprocess from + # any PYTHON_THREAD_INHERIT_CONTEXT in the test environment. + script_helper.assert_python_ok( + "-W", "error::DeprecationWarning", + "-X", "context_aware_warnings=1", "-c", code, + PYTHON_THREAD_INHERIT_CONTEXT="") + + def test_private_thread_start_context(self): + import _contextvars + + self.assertFalse(hasattr(contextvars, "_thread_start_context")) + ctx = _contextvars._thread_start_context() + self.assertIs(type(ctx), contextvars.Context) + self.assertEqual(ctx.run(lambda: "result"), "result") + def test_token_contextmanager_with_default(self): ctx = contextvars.Context() c = contextvars.ContextVar('c', default=42) @@ -586,6 +857,355 @@ def __eq__(self, other): ctx2.run(var.set, ReentrantHash()) ctx1 == ctx2 + def test_context_depth_increases_on_run(self): + # Entering a copied context reports a depth one greater than the + # context it was copied from; the depth is restored after the run. + from _contextvars import _current_context_depth as depth + base = depth() + got = [] + contextvars.copy_context().run(lambda: got.append(depth())) + self.assertEqual(got, [base + 1]) + self.assertEqual(depth(), base) + + def test_context_depth_nested_run(self): + # Nested copied contexts increase the depth by one per level. + from _contextvars import _current_context_depth as depth + base = depth() + got = [] + + def outer(): + got.append(depth()) + contextvars.copy_context().run( + lambda: got.append(depth())) + contextvars.copy_context().run(outer) + self.assertEqual(got, [base + 1, base + 2]) + + def test_context_depth_reenter_same_context(self): + # The depth is fixed when the context is created, so running the + # same context object again reports the same depth. + from _contextvars import _current_context_depth as depth + ctx = contextvars.copy_context() + got = [] + ctx.run(lambda: got.append(depth())) + ctx.run(lambda: got.append(depth())) + self.assertEqual(got[0], got[1]) + + def test_context_depth_copy_method(self): + # Context.copy() produces a context one level deeper than its source. + from _contextvars import _current_context_depth as depth + base = depth() + # copy_context() -> depth base+1; .copy() of that -> depth base+2, + # regardless of where it is entered (depth is a creation property). + ctx = contextvars.copy_context().copy() + got = [] + ctx.run(lambda: got.append(depth())) + self.assertEqual(got, [base + 2]) + + def test_context_depth_empty_context_is_zero(self): + # A freshly created (empty) Context has depth 0 and reports it + # independently of the context it is entered from. + from _contextvars import _current_context_depth as depth + got = [] + contextvars.Context().run(lambda: got.append(depth())) + # Entered from within a deeper context, still its own depth (0). + contextvars.copy_context().run( + lambda: contextvars.Context().run( + lambda: got.append(depth()))) + self.assertEqual(got, [0, 0]) + + def test_context_depth_inherited_value_is_shared(self): + # A value set before copying is inherited by the copied context as + # the same object, while the depth differs -- this is the signal a + # mutable value (e.g. a decimal context) uses to copy for isolation. + from _contextvars import _current_context_depth as depth + v = contextvars.ContextVar('v') + sentinel = object() + v.set(sentinel) + base = depth() + ctx = contextvars.copy_context() + + def check(): + self.assertEqual(depth(), base + 1) + self.assertIs(v.get(), sentinel) + ctx.run(check) + + @threading_helper.requires_working_threading() + def test_context_depth_with_threads(self): + # A thread running a copied context sees a deeper context than the + # parent, and each thread's depth is independent. + import threading + from _contextvars import _current_context_depth as depth + base = depth() + results = {} + + def thread_func(name): + results[name] = depth() + + t1 = threading.Thread(target=contextvars.copy_context().run, + args=(lambda: thread_func('t1'),)) + t2 = threading.Thread(target=contextvars.copy_context().run, + args=(lambda: thread_func('t2'),)) + t1.start(); t2.start() + t1.join(); t2.join() + self.assertEqual(results['t1'], base + 1) + self.assertEqual(results['t2'], base + 1) + + +@threading_helper.requires_working_threading() +class ThreadInheritableVarTest(unittest.TestCase): + # These tests run in a subprocess with -X thread_inherit_context pinned, + # since its default depends on the build (true on free-threaded builds). + + def run_with_flag(self, flag, source): + _, _, stderr = script_helper.assert_python_ok( + '-X', f'thread_inherit_context={flag}', '-c', source) + self.assertEqual(stderr, b'') + + def test_thread_inheritance(self): + self.run_with_flag(0, """if True: + import threading + from contextvars import ContextVar, copy_context + + inh = ContextVar.thread_inheritable('inh', default='default') + plain = ContextVar('plain') + inh.set('inherited') + plain.set('not inherited') + + def child(): + # The binding is a real binding in the thread's context: + # visible to get(), copy_context() and Context methods. + assert inh.get() == 'inherited' + ctx = copy_context() + assert inh in ctx + assert ctx[inh] == 'inherited' + assert ctx.run(inh.get) == 'inherited' + # Non-inheritable vars are not visible. + assert plain not in ctx + try: + plain.get() + except LookupError: + pass + else: + raise AssertionError('plain was inherited') + + t = threading.Thread(target=child) + t.start() + t.join() + """) + + def test_inheritance_captured_at_start_and_context_copy(self): + self.run_with_flag(0, """if True: + import threading + from contextvars import Context, ContextVar + + inh = ContextVar.thread_inheritable('inh') + values = [] + + # The binding is captured by start(), not by Thread(). + t = threading.Thread(target=lambda: values.append(inh.get())) + inh.set('at start') + t.start() + t.join() + + def start_and_join(): + t = threading.Thread( + target=lambda: values.append(inh.get())) + t.start() + t.join() + + # Context.run() and Context.copy() preserve the inheritable subset. + ctx = Context() + ctx.run(inh.set, 'context') + ctx.run(start_and_join) + ctx_copy = ctx.copy() + ctx_copy.run(inh.set, 'copy') + ctx_copy.run(start_and_join) + + assert values == ['at start', 'context', 'copy'] + """) + + def test_thread_start_retry_recaptures_context(self): + self.run_with_flag(0, """if True: + import threading + from contextvars import ContextVar + + inh = ContextVar.thread_inheritable('inh') + inh.set('first attempt') + values = [] + t = threading.Thread(target=lambda: values.append(inh.get())) + + start_joinable_thread = threading._start_joinable_thread + + def fail_start(*args, **kwargs): + raise threading.ThreadError + + threading._start_joinable_thread = fail_start + try: + try: + t.start() + except threading.ThreadError: + pass + else: + raise AssertionError('thread start did not fail') + finally: + threading._start_joinable_thread = start_joinable_thread + + inh.set('retry') + t.start() + t.join() + assert values == ['retry'] + """) + + def test_thread_set_and_reset(self): + self.run_with_flag(0, """if True: + import threading + from contextvars import ContextVar + + inh = ContextVar.thread_inheritable('inh') + inh.set('inherited') + + def child(): + token = inh.set('child value') + assert inh.get() == 'child value' + inh.reset(token) + assert inh.get() == 'inherited' + + t = threading.Thread(target=child) + t.start() + t.join() + # The thread's set() does not affect the parent. + assert inh.get() == 'inherited' + """) + + def test_thread_inheritance_transitive(self): + self.run_with_flag(0, """if True: + import threading + from contextvars import ContextVar + + inh = ContextVar.thread_inheritable('inh') + inh.set('inherited') + + def grandchild(): + assert inh.get() == 'inherited' + + def child(): + # The child never sets the var; the binding must still + # propagate to threads it starts. + t = threading.Thread(target=grandchild) + t.start() + t.join() + + t = threading.Thread(target=child) + t.start() + t.join() + """) + + def test_thread_inheritance_unset_or_deleted(self): + self.run_with_flag(0, """if True: + import threading + from contextvars import ContextVar + + unset = ContextVar.thread_inheritable('unset', default='default') + deleted = ContextVar.thread_inheritable('deleted') + token = deleted.set('inherited') + deleted.reset(token) + + def child(): + assert unset.get() == 'default' + try: + deleted.get() + except LookupError: + pass + else: + raise AssertionError('deleted binding was inherited') + + t = threading.Thread(target=child) + t.start() + t.join() + """) + + def test_thread_explicit_context(self): + self.run_with_flag(0, """if True: + import threading + from contextvars import ContextVar, Context + + inh = ContextVar.thread_inheritable('inh', default='default') + inh.set('inherited') + + def child(): + assert inh.get() == 'default' + + t = threading.Thread(target=child, context=Context()) + t.start() + t.join() + """) + + def test_thread_inheritance_asyncio(self): + self.run_with_flag(0, """if True: + import asyncio + import threading + from contextvars import ContextVar + + inh = ContextVar.thread_inheritable('inh') + plain = ContextVar('plain') + inh.set('inherited') + plain.set('not inherited') + + def check_plain_not_set(): + try: + plain.get() + except LookupError: + pass + else: + raise AssertionError('plain was inherited') + + async def task(): + # Tasks run in a copy of the thread's context, which + # includes the inherited binding. + assert inh.get() == 'inherited' + check_plain_not_set() + # A set() inside the task is confined to the task. + inh.set('task value') + + async def main(): + assert inh.get() == 'inherited' + await asyncio.gather(task(), task()) + assert inh.get() == 'inherited' + # Callbacks also run in a copy of the current context. + loop = asyncio.get_running_loop() + fut = loop.create_future() + loop.call_soon( + lambda: fut.set_result((inh.get(), plain.get(None)))) + assert await fut == ('inherited', None) + + def child(): + asyncio.run(main()) + + t = threading.Thread(target=child) + t.start() + t.join() + """) + + def test_thread_inherit_context_flag_true(self): + self.run_with_flag(1, """if True: + import threading + from contextvars import ContextVar + + inh = ContextVar.thread_inheritable('inh') + plain = ContextVar('plain') + inh.set('inherited') + plain.set('also inherited') + + def child(): + # With the flag set, the full context is copied. + assert inh.get() == 'inherited' + assert plain.get() == 'also inherited' + + t = threading.Thread(target=child) + t.start() + t.join() + """) + # HAMT Tests diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 1c723b25784da1..35999a8c3e0dcd 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -1770,6 +1770,101 @@ def test_threading(self): DefaultContext.Emax = save_emax DefaultContext.Emin = save_emin + @threading_helper.requires_working_threading() + def test_inherited_context_isolation(self): + # Test that when threads inherit contextvars (e.g. via + # sys.flags.thread_inherit_context), each thread gets its own + # copy of the decimal context so mutations don't leak between + # threads. Also verifies correct behavior with asyncio tasks. + Decimal = self.decimal.Decimal + getcontext = self.decimal.getcontext + setcontext = self.decimal.setcontext + Context = self.decimal.Context + Underflow = self.decimal.Underflow + + # Set up parent context with specific precision + parent_ctx = getcontext() + parent_ctx.prec = 20 + + barrier = threading.Barrier(2, timeout=2) + results = {} + + def child(name, prec_delta): + barrier.wait() + ctx = getcontext() + # Each child should see a context with the parent's precision + results[name + '_initial_prec'] = ctx.prec + # Keep the context alive until both threads have finished so an + # allocator-reused address cannot make distinct contexts appear + # to have the same identity. + results[name + '_ctx'] = ctx + # Mutate this thread's context + ctx.prec += prec_delta + results[name + '_modified_prec'] = ctx.prec + + # Spawn threads that inherit the parent's contextvars. + t1 = threading.Thread(target=child, args=('t1', 5), + context=contextvars.copy_context()) + t2 = threading.Thread(target=child, args=('t2', 10), + context=contextvars.copy_context()) + t1.start() + t2.start() + t1.join() + t2.join() + + # Each thread should have started with the parent's precision + self.assertEqual(results['t1_initial_prec'], 20) + self.assertEqual(results['t2_initial_prec'], 20) + + # Each thread should have its own context. + self.assertIsNot(results['t1_ctx'], results['t2_ctx']) + + # Mutations should be independent + self.assertEqual(results['t1_modified_prec'], 25) + self.assertEqual(results['t2_modified_prec'], 30) + + # Parent context should be unaffected + self.assertEqual(getcontext().prec, 20) + + def test_inherited_context_isolation_async(self): + # An asyncio child task inherits the parent task's context object + # (create_task copies the current context). Each task must get its + # own decimal context so mutations stay isolated. This is the case + # where every task step runs at the same context nesting level, so a + # per-entry depth would collide -- the depth is assigned when the + # context is *copied*, which keeps parent and child distinct. + import asyncio + getcontext = self.decimal.getcontext + + async def child(results): + ctx = getcontext() + results['child_initial_prec'] = ctx.prec + results['child_ctx_id'] = id(ctx) + ctx.prec = 7 + results['child_modified_prec'] = ctx.prec + + async def parent(): + results = {} + ctx = getcontext() + ctx.prec = 33 + results['parent_ctx_id'] = id(ctx) + await asyncio.create_task(child(results)) + results['parent_after_prec'] = getcontext().prec + return results + + # Pass loop_factory so asyncio.run() doesn't lazily initialize the + # global event loop policy, which would be reported as "env changed" + # by regrtest (e.g. on iOS/Android where the policy starts as None). + results = asyncio.run(parent(), loop_factory=asyncio.EventLoop) + + # Child inherits the parent's precision value... + self.assertEqual(results['child_initial_prec'], 33) + # ...but in its own context object. + self.assertNotEqual(results['parent_ctx_id'], results['child_ctx_id']) + # The child's mutation does not leak back to the parent. + self.assertEqual(results['child_modified_prec'], 7) + self.assertEqual(results['parent_after_prec'], 33) + @requires_cdecimal class CThreadingTest(ThreadingTest, unittest.TestCase): diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index bf1bcf8e6ed5d9..6398af0322d1ac 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -1807,12 +1807,11 @@ class PyAsyncTests(AsyncTests, unittest.TestCase): class ThreadTests(BaseTest): """Verifies that the catch_warnings() context manager behaves as - expected when used within threads. This requires that both the - context_aware_warnings flag and thread_inherit_context flags are enabled. + expected when used within threads. This requires the + context_aware_warnings flag to be enabled. """ - ENABLE_THREAD_TESTS = (sys.flags.context_aware_warnings and - sys.flags.thread_inherit_context) + ENABLE_THREAD_TESTS = sys.flags.context_aware_warnings def setUp(self): super().setUp() diff --git a/Lib/threading.py b/Lib/threading.py index abac31e25886fa..3f33b296849227 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -1034,8 +1034,9 @@ class is implemented. *context* is the contextvars.Context value to use for the thread. The default value is None, which means to check sys.flags.thread_inherit_context. If that flag is true, use a copy - of the context of the caller. If false, use an empty context. To - explicitly start with an empty context, pass a new instance of + of the context of the caller. If false, use a context containing only + the bindings of thread-inheritable context variables. To explicitly + start with an empty context, pass a new instance of contextvars.Context(). To explicitly start with a copy of the current context, pass the value from contextvars.copy_context(). @@ -1127,14 +1128,17 @@ def start(self): with _active_limbo_lock: _limbo[self] = self - if self._context is None: + context_is_implicit = self._context is None + if context_is_implicit: # No context provided if _sys.flags.thread_inherit_context: # start with a copy of the context of the caller self._context = _contextvars.copy_context() else: - # start with an empty context - self._context = _contextvars.Context() + # Start with a context containing only the bindings of + # thread-inheritable context variables (see + # ContextVar.thread_inheritable); usually empty. + self._context = _contextvars._thread_start_context() try: # Start joinable thread @@ -1143,6 +1147,9 @@ def start(self): except Exception: with _active_limbo_lock: del _limbo[self] + if context_is_implicit: + # Capture the caller's context again if start() is retried. + self._context = None raise self._started.wait() # Will set ident and native_id @@ -1218,7 +1225,12 @@ def _bootstrap_inner(self): self._context.run(self.run) except: self._invoke_excepthook(self) + finally: + # Free context, we don't need it anymore. + self._context = None finally: + # Break references held by the context after any bootstrap path. + self._context = None self._delete() def _delete(self): diff --git a/Misc/NEWS.d/next/Library/2026-03-26-10-27-07.gh-issue-141148._XpYnI.rst b/Misc/NEWS.d/next/Library/2026-03-26-10-27-07.gh-issue-141148._XpYnI.rst new file mode 100644 index 00000000000000..8589be19132a74 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-26-10-27-07.gh-issue-141148._XpYnI.rst @@ -0,0 +1,3 @@ +Ensure that :func:`decimal.getcontext` returns a per-task copy of the +:class:`decimal.Context` so that mutations are isolated between asyncio +tasks and threads using :data:`sys.flags.thread_inherit_context`. diff --git a/Misc/NEWS.d/next/Library/2026-07-23-10-16-59.gh-issue-154549.Ulgmsd.rst b/Misc/NEWS.d/next/Library/2026-07-23-10-16-59.gh-issue-154549.Ulgmsd.rst new file mode 100644 index 00000000000000..4d82c8feac0fbd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-23-10-16-59.gh-issue-154549.Ulgmsd.rst @@ -0,0 +1,5 @@ +On GIL-enabled builds, emit a :exc:`DeprecationWarning` when the default +implicitly empty :class:`threading.Thread` context causes a context variable +lookup to differ from the context that a future release will inherit by +default. Setting ``thread_inherit_context`` selects the desired behavior +without a warning. diff --git a/Misc/NEWS.d/next/Library/2026-07-23-13-19-38.gh-issue-154562.WBaAJV.rst b/Misc/NEWS.d/next/Library/2026-07-23-13-19-38.gh-issue-154562.WBaAJV.rst new file mode 100644 index 00000000000000..08def726565d89 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-23-13-19-38.gh-issue-154562.WBaAJV.rst @@ -0,0 +1,6 @@ +Add :meth:`contextvars.ContextVar.thread_inheritable`, an alternative +constructor for context variables whose bindings are inherited by new +:class:`threading.Thread` instances even when +:data:`sys.flags.thread_inherit_context` is false. This lets libraries opt +individual context variables in to thread inheritance without requiring the +application to enable the flag process-wide. diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 6fbce0ed85e695..184a01e6da93bf 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -30,6 +30,7 @@ #endif #include +#include "pycore_context.h" // _PyContext_CurrentDepth() #include "pycore_object.h" // _PyObject_VisitType() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_tuple.h" // _PyTuple_FromPair @@ -224,6 +225,11 @@ typedef struct PyDecContextObject { int capitals; PyThreadState *tstate; decimal_state *modstate; + /* Depth of the contextvars context this context object was bound into + (see _pycontextobject.ctx_depth). Used to detect when the current + context object was inherited from an outer context/task and therefore + must be copied to keep mutations isolated. */ + uint64_t ctx_depth; } PyDecContextObject; #define _PyDecContextObject_CAST(op) ((PyDecContextObject *)(op)) @@ -247,6 +253,7 @@ typedef struct { #define SdFlags(v) (*_PyDecSignalDictObject_CAST(v)->flags) #define CTX(v) (&_PyDecContextObject_CAST(v)->ctx) #define CtxCaps(v) (_PyDecContextObject_CAST(v)->capitals) +#define CtxDepth(v) (_PyDecContextObject_CAST(v)->ctx_depth) static inline decimal_state * get_module_state_from_ctx(PyObject *v) @@ -1477,6 +1484,7 @@ context_new(PyTypeObject *type, CtxCaps(self) = 1; self->tstate = NULL; self->modstate = state; + self->ctx_depth = 0; if (type == state->PyDecContext_Type) { PyObject_GC_Track(self); @@ -1929,13 +1937,17 @@ PyDec_SetCurrentContext(PyObject *self, PyObject *v) } #else static PyObject * -init_current_context(decimal_state *state) +init_current_context(decimal_state *state, PyObject *prev_context, + uint64_t depth) { - PyObject *tl_context = context_copy(state, state->default_context_template); + PyObject *tl_context = context_copy(state, prev_context); if (tl_context == NULL) { return NULL; } CTX(tl_context)->status = 0; + /* Stamp the copy with the current context's depth so that subsequent + lookups in this same context recognize it as locally owned. */ + CtxDepth(tl_context) = depth; PyObject *tok = PyContextVar_Set(state->current_context_var, tl_context); if (tok == NULL) { @@ -1955,11 +1967,24 @@ current_context(decimal_state *state) return NULL; } + uint64_t cur_depth = _PyContext_CurrentDepth(); + if (tl_context != NULL) { - return tl_context; + if (CtxDepth(tl_context) == cur_depth) { + /* The context object was created for this same context scope. */ + return tl_context; + } + /* The context object was inherited from an outer context (e.g. a + parent thread or asyncio task); copy it so that mutations stay + isolated from the context that shared it. */ + PyObject *new_context = init_current_context(state, tl_context, + cur_depth); + Py_DECREF(tl_context); + return new_context; } - return init_current_context(state); + return init_current_context(state, state->default_context_template, + cur_depth); } /* ctxobj := borrowed reference to the current context */ @@ -2002,6 +2027,10 @@ PyDec_SetCurrentContext(PyObject *self, PyObject *v) Py_INCREF(v); } + /* Mark the context object as owned by the current context scope, so a + following getcontext() returns this very object rather than a copy. */ + CtxDepth(v) = _PyContext_CurrentDepth(); + PyObject *tok = PyContextVar_Set(state->current_context_var, v); Py_DECREF(v); if (tok == NULL) { @@ -7902,7 +7931,7 @@ _decimal_exec(PyObject *m) PyUnicode_FromString("___DECIMAL_CTX__")); CHECK_INT(PyModule_AddObjectRef(m, "HAVE_CONTEXTVAR", Py_False)); #else - ASSIGN_PTR(state->current_context_var, PyContextVar_New("decimal_context", NULL)); + ASSIGN_PTR(state->current_context_var, PyContextVar_NewThreadInheritable("decimal_context", NULL)); CHECK_INT(PyModule_AddObjectRef(m, "HAVE_CONTEXTVAR", Py_True)); #endif CHECK_INT(PyModule_AddObjectRef(m, "HAVE_THREADS", Py_True)); diff --git a/Python/_contextvars.c b/Python/_contextvars.c index 86acc94fbc79cd..a6dc060ca1d621 100644 --- a/Python/_contextvars.c +++ b/Python/_contextvars.c @@ -1,4 +1,5 @@ #include "Python.h" +#include "pycore_context.h" // _PyContext_*() #include "clinic/_contextvars.c.h" @@ -20,10 +21,36 @@ _contextvars_copy_context_impl(PyObject *module) } +/*[clinic input] +_contextvars._thread_start_context +[clinic start generated code]*/ + +static PyObject * +_contextvars__thread_start_context_impl(PyObject *module) +/*[clinic end generated code: output=7e656d156c385a65 input=4575c5d2223de58d]*/ +{ + return _PyContext_NewForThread(); +} + + +/* Return the depth of the current context (see _pycontextobject.ctx_depth). + Used by the pure-Python decimal implementation to detect when a context + object was inherited from an outer context/task. Private API. */ +static PyObject * +_contextvars_current_context_depth(PyObject *module, + PyObject *Py_UNUSED(ignored)) +{ + return PyLong_FromUnsignedLongLong(_PyContext_CurrentDepth()); +} + + PyDoc_STRVAR(module_doc, "Context Variables"); static PyMethodDef _contextvars_methods[] = { _CONTEXTVARS_COPY_CONTEXT_METHODDEF + _CONTEXTVARS__THREAD_START_CONTEXT_METHODDEF + {"_current_context_depth", _contextvars_current_context_depth, + METH_NOARGS, NULL}, {NULL, NULL} }; diff --git a/Python/_warnings.c b/Python/_warnings.c index 4f6de50efa14a8..45a357e07278dc 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -158,7 +158,8 @@ _PyWarnings_InitState(PyInterpreterState *interp) } if (st->context == NULL) { - st->context = PyContextVar_New("_warnings_context", NULL); + st->context = PyContextVar_NewThreadInheritable( + "_warnings_context", NULL); if (st->context == NULL) { return -1; } diff --git a/Python/clinic/_contextvars.c.h b/Python/clinic/_contextvars.c.h index b1885e41c355d2..381d8c159beb4b 100644 --- a/Python/clinic/_contextvars.c.h +++ b/Python/clinic/_contextvars.c.h @@ -18,4 +18,21 @@ _contextvars_copy_context(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _contextvars_copy_context_impl(module); } -/*[clinic end generated code: output=26e07024451baf52 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_contextvars__thread_start_context__doc__, +"_thread_start_context($module, /)\n" +"--\n" +"\n"); + +#define _CONTEXTVARS__THREAD_START_CONTEXT_METHODDEF \ + {"_thread_start_context", (PyCFunction)_contextvars__thread_start_context, METH_NOARGS, _contextvars__thread_start_context__doc__}, + +static PyObject * +_contextvars__thread_start_context_impl(PyObject *module); + +static PyObject * +_contextvars__thread_start_context(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _contextvars__thread_start_context_impl(module); +} +/*[clinic end generated code: output=2c6c9d89faff4312 input=a9049054013a1b77]*/ diff --git a/Python/clinic/context.c.h b/Python/clinic/context.c.h index ece7341d65d5fb..f8a27bf5b8ab64 100644 --- a/Python/clinic/context.c.h +++ b/Python/clinic/context.c.h @@ -2,6 +2,10 @@ preserve [clinic start generated code]*/ +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_contextvars_Context_get__doc__, @@ -116,6 +120,77 @@ _contextvars_Context_copy(PyObject *self, PyObject *Py_UNUSED(ignored)) return _contextvars_Context_copy_impl((PyContext *)self); } +PyDoc_STRVAR(_contextvars_ContextVar_thread_inheritable__doc__, +"thread_inheritable($type, name, /, *, default=)\n" +"--\n" +"\n" +"Create a context variable whose binding is inherited by new threads.\n" +"\n" +"The bindings of such variables are copied into the context of a new\n" +"thread by threading.Thread.start() when the thread would otherwise\n" +"start with an empty context."); + +#define _CONTEXTVARS_CONTEXTVAR_THREAD_INHERITABLE_METHODDEF \ + {"thread_inheritable", _PyCFunction_CAST(_contextvars_ContextVar_thread_inheritable), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, _contextvars_ContextVar_thread_inheritable__doc__}, + +static PyObject * +_contextvars_ContextVar_thread_inheritable_impl(PyTypeObject *type, + PyObject *name, + PyObject *default_value); + +static PyObject * +_contextvars_ContextVar_thread_inheritable(PyObject *type, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(default), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"", "default", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "thread_inheritable", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *name; + PyObject *default_value = NULL; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + name = args[0]; + if (!noptargs) { + goto skip_optional_kwonly; + } + default_value = args[1]; +skip_optional_kwonly: + return_value = _contextvars_ContextVar_thread_inheritable_impl((PyTypeObject *)type, name, default_value); + +exit: + return return_value; +} + PyDoc_STRVAR(_contextvars_ContextVar_get__doc__, "get($self, default=, /)\n" "--\n" @@ -259,4 +334,4 @@ token_exit(PyObject *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=90ec3e4375804e9b input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d8a9c933c7fcf8c3 input=a9049054013a1b77]*/ diff --git a/Python/context.c b/Python/context.c index 4678054ff3ad74..2f101d6cc81def 100644 --- a/Python/context.c +++ b/Python/context.c @@ -47,7 +47,9 @@ static PyContext * context_new_empty(void); static PyContext * -context_new_from_vars(PyHamtObject *vars); +context_new_from_vars(PyHamtObject *vars, + PyHamtObject *thread_inheritable_vars, + uint64_t depth); static inline PyContext * context_get(void); @@ -56,7 +58,7 @@ static PyContextToken * token_new(PyContext *ctx, PyContextVar *var, PyObject *val); static PyContextVar * -contextvar_new(PyObject *name, PyObject *def); +contextvar_new(PyObject *name, PyObject *def, int thread_inheritable); static int contextvar_set(PyContextVar *var, PyObject *val); @@ -79,12 +81,45 @@ PyContext_New(void) } +PyObject * +_PyContext_NewForThread(void) +{ + // The thread-inheritable subset becomes the new context's full vars map. + // Every entry in it is thread-inheritable, so it is also its own subset. + PyContext *starter_ctx = context_get(); + PyContext *ctx = context_new_from_vars( + starter_ctx->ctx_thread_inheritable_vars, + starter_ctx->ctx_thread_inheritable_vars, + starter_ctx->ctx_depth + 1); + if (ctx == NULL) { + return NULL; + } + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (interp->thread_inherit_context_warn) { + // set ctx_starter_vars if we need to emit warnings + PyThreadState *ts = _PyThreadState_GET(); + assert(ts != NULL); + PyContext *starter_ctx = (PyContext *)ts->context; + if (starter_ctx != NULL && _PyHamt_Len(starter_ctx->ctx_vars) > 0) { + // Need at least one variable set in the starter context so a lookup + // in the new thread can differ from the inheriting behavior; + // otherwise a snapshot would only add overhead to ContextVar.get() + // misses. + ctx->ctx_starter_vars = (PyHamtObject*)Py_NewRef(starter_ctx->ctx_vars); + } + } + return (PyObject *)ctx; +} + + PyObject * PyContext_Copy(PyObject * octx) { ENSURE_Context(octx, NULL) PyContext *ctx = (PyContext *)octx; - return (PyObject *)context_new_from_vars(ctx->ctx_vars); + return (PyObject *)context_new_from_vars( + ctx->ctx_vars, ctx->ctx_thread_inheritable_vars, + ctx->ctx_depth + 1); } @@ -96,7 +131,9 @@ PyContext_CopyCurrent(void) return NULL; } - return (PyObject *)context_new_from_vars(ctx->ctx_vars); + return (PyObject *)context_new_from_vars( + ctx->ctx_vars, ctx->ctx_thread_inheritable_vars, + ctx->ctx_depth + 1); } static const char * @@ -262,18 +299,31 @@ PyContext_Exit(PyObject *octx) } -PyObject * -PyContextVar_New(const char *name, PyObject *def) +static PyObject * +contextvar_new_from_utf8(const char *name, PyObject *def, + int thread_inheritable) { PyObject *pyname = PyUnicode_FromString(name); if (pyname == NULL) { return NULL; } - PyContextVar *var = contextvar_new(pyname, def); + PyContextVar *var = contextvar_new(pyname, def, thread_inheritable); Py_DECREF(pyname); return (PyObject *)var; } +PyObject * +PyContextVar_New(const char *name, PyObject *def) +{ + return contextvar_new_from_utf8(name, def, 0); +} + +PyObject * +PyContextVar_NewThreadInheritable(const char *name, PyObject *def) +{ + return contextvar_new_from_utf8(name, def, 1); +} + int PyContextVar_Get(PyObject *ovar, PyObject *def, PyObject **val) @@ -298,7 +348,8 @@ PyContextVar_Get(PyObject *ovar, PyObject *def, PyObject **val) #endif assert(PyContext_CheckExact(ts->context)); - PyHamtObject *vars = ((PyContext *)ts->context)->ctx_vars; + PyContext *ctx = (PyContext *)ts->context; + PyHamtObject *vars = ctx->ctx_vars; PyObject *found = NULL; int res = _PyHamt_Find(vars, (PyObject*)var, &found); @@ -317,6 +368,27 @@ PyContextVar_Get(PyObject *ovar, PyObject *def, PyObject **val) goto found; } + if (ctx->ctx_starter_vars != NULL) { + res = _PyHamt_Find(ctx->ctx_starter_vars, (PyObject *)var, &found); + if (res < 0) { + goto error; + } + if (res == 1) { + // Detach before warning so that warning machinery using context + // variables cannot recursively warn. This also means we warn + // once per thread. + Py_CLEAR(ctx->ctx_starter_vars); + if (PyErr_WarnEx( + PyExc_DeprecationWarning, + "threads will inherit context by default in a future " + "Python release", + 1) < 0) + { + goto error; + } + } + } + not_found: if (def == NULL) { if (var->var_default != NULL) { @@ -413,6 +485,19 @@ PyContextVar_Reset(PyObject *ovar, PyObject *otok) } +uint64_t +_PyContext_CurrentDepth(void) +{ + PyThreadState *ts = _PyThreadState_GET(); + assert(ts != NULL); + if (ts->context == NULL) { + return 0; + } + assert(PyContext_CheckExact(ts->context)); + return ((PyContext *)ts->context)->ctx_depth; +} + + /////////////////////////// PyContext /*[clinic input] @@ -439,6 +524,9 @@ _context_alloc(void) ctx->ctx_prev = NULL; ctx->ctx_entered = 0; ctx->ctx_weakreflist = NULL; + ctx->ctx_thread_inheritable_vars = NULL; + ctx->ctx_starter_vars = NULL; + ctx->ctx_depth = 0; return ctx; } @@ -458,13 +546,21 @@ context_new_empty(void) return NULL; } + ctx->ctx_thread_inheritable_vars = _PyHamt_New(); + if (ctx->ctx_thread_inheritable_vars == NULL) { + Py_DECREF(ctx); + return NULL; + } + _PyObject_GC_TRACK(ctx); return ctx; } static PyContext * -context_new_from_vars(PyHamtObject *vars) +context_new_from_vars(PyHamtObject *vars, + PyHamtObject *thread_inheritable_vars, + uint64_t depth) { PyContext *ctx = _context_alloc(); if (ctx == NULL) { @@ -472,6 +568,9 @@ context_new_from_vars(PyHamtObject *vars) } ctx->ctx_vars = (PyHamtObject*)Py_NewRef(vars); + ctx->ctx_thread_inheritable_vars = + (PyHamtObject*)Py_NewRef(thread_inheritable_vars); + ctx->ctx_depth = depth; _PyObject_GC_TRACK(ctx); return ctx; @@ -523,6 +622,8 @@ context_tp_clear(PyObject *op) PyContext *self = _PyContext_CAST(op); Py_CLEAR(self->ctx_prev); Py_CLEAR(self->ctx_vars); + Py_CLEAR(self->ctx_thread_inheritable_vars); + Py_CLEAR(self->ctx_starter_vars); return 0; } @@ -532,6 +633,8 @@ context_tp_traverse(PyObject *op, visitproc visit, void *arg) PyContext *self = _PyContext_CAST(op); Py_VISIT(self->ctx_prev); Py_VISIT(self->ctx_vars); + Py_VISIT(self->ctx_thread_inheritable_vars); + Py_VISIT(self->ctx_starter_vars); return 0; } @@ -708,7 +811,9 @@ static PyObject * _contextvars_Context_copy_impl(PyContext *self) /*[clinic end generated code: output=30ba8896c4707a15 input=ebafdbdd9c72d592]*/ { - return (PyObject *)context_new_from_vars(self->ctx_vars); + return (PyObject *)context_new_from_vars( + self->ctx_vars, self->ctx_thread_inheritable_vars, + self->ctx_depth + 1); } @@ -801,7 +906,23 @@ contextvar_set(PyContextVar *var, PyObject *val) return -1; } + // Compute both new maps before installing either so that an error + // leaves the context unchanged. + PyHamtObject *new_thread_inheritable_vars = NULL; + if (var->var_thread_inheritable) { + new_thread_inheritable_vars = _PyHamt_Assoc( + ctx->ctx_thread_inheritable_vars, (PyObject *)var, val); + if (new_thread_inheritable_vars == NULL) { + Py_DECREF(new_vars); + return -1; + } + } + Py_SETREF(ctx->ctx_vars, new_vars); + if (new_thread_inheritable_vars != NULL) { + Py_SETREF(ctx->ctx_thread_inheritable_vars, + new_thread_inheritable_vars); + } #ifndef Py_GIL_DISABLED var->var_cached = val; /* borrow */ @@ -835,7 +956,23 @@ contextvar_del(PyContextVar *var) return -1; } + // Compute both new maps before installing either so that an error + // leaves the context unchanged. + PyHamtObject *new_thread_inheritable_vars = NULL; + if (var->var_thread_inheritable) { + new_thread_inheritable_vars = _PyHamt_Without( + ctx->ctx_thread_inheritable_vars, (PyObject *)var); + if (new_thread_inheritable_vars == NULL) { + Py_DECREF(new_vars); + return -1; + } + } + Py_SETREF(ctx->ctx_vars, new_vars); + if (new_thread_inheritable_vars != NULL) { + Py_SETREF(ctx->ctx_thread_inheritable_vars, + new_thread_inheritable_vars); + } return 0; } @@ -868,7 +1005,7 @@ contextvar_generate_hash(void *addr, PyObject *name) } static PyContextVar * -contextvar_new(PyObject *name, PyObject *def) +contextvar_new(PyObject *name, PyObject *def, int thread_inheritable) { if (!PyUnicode_Check(name)) { PyErr_SetString(PyExc_TypeError, @@ -883,6 +1020,7 @@ contextvar_new(PyObject *name, PyObject *def) var->var_name = Py_NewRef(name); var->var_default = Py_XNewRef(def); + var->var_thread_inheritable = thread_inheritable; #ifndef Py_GIL_DISABLED var->var_cached = NULL; @@ -927,7 +1065,7 @@ contextvar_tp_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } - return (PyObject *)contextvar_new(name, def); + return (PyObject *)contextvar_new(name, def, 0); } static int @@ -1009,6 +1147,32 @@ contextvar_tp_repr(PyObject *op) } +/*[clinic input] +@classmethod +_contextvars.ContextVar.thread_inheritable + name: object + / + * + default: object = NULL + +Create a context variable whose binding is inherited by new threads. + +The bindings of such variables are copied into the context of a new +thread by threading.Thread.start() when the thread would otherwise +start with an empty context. +[clinic start generated code]*/ + +static PyObject * +_contextvars_ContextVar_thread_inheritable_impl(PyTypeObject *type, + PyObject *name, + PyObject *default_value) +/*[clinic end generated code: output=a890265ff610a979 input=f211f3eedeb507b8]*/ +{ + assert(type == &PyContextVar_Type); + return (PyObject *)contextvar_new(name, default_value, 1); +} + + /*[clinic input] _contextvars.ContextVar.get default: object = NULL @@ -1099,6 +1263,7 @@ static PyMemberDef PyContextVar_members[] = { }; static PyMethodDef PyContextVar_methods[] = { + _CONTEXTVARS_CONTEXTVAR_THREAD_INHERITABLE_METHODDEF _CONTEXTVARS_CONTEXTVAR_GET_METHODDEF _CONTEXTVARS_CONTEXTVAR_SET_METHODDEF _CONTEXTVARS_CONTEXTVAR_RESET_METHODDEF diff --git a/Python/initconfig.c b/Python/initconfig.c index b9bacb17a66454..75d87b7fe798c8 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -2105,6 +2105,21 @@ config_init_cpu_count(PyConfig *config) "n must be greater than 0"); } +// Return true if we should warn about threads *not* inheriting context +// from the starting thread. +int +_PyConfig_ThreadInheritContextWarn(const PyConfig *config) +{ +#ifdef Py_GIL_DISABLED + (void)config; + return 0; +#else + return (config->thread_inherit_context == 0 + && config_get_env(config, "PYTHON_THREAD_INHERIT_CONTEXT") == NULL + && config_get_xoption(config, L"thread_inherit_context") == NULL); +#endif +} + static PyStatus config_init_thread_inherit_context(PyConfig *config) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 01ca459b2eb2b8..a8d15ef3e43f10 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -479,6 +479,8 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime, if (_PyStatus_EXCEPTION(status)) { return status; } + interp->thread_inherit_context_warn = + _PyConfig_ThreadInheritContextWarn(config); config = _PyInterpreterState_GetConfig(interp); if (config->_install_importlib) { @@ -691,6 +693,8 @@ pycore_create_interpreter(_PyRuntimeState *runtime, if (_PyStatus_EXCEPTION(status)) { return status; } + interp->thread_inherit_context_warn = + _PyConfig_ThreadInheritContextWarn(src_config); /* Auto-thread-state API */ status = _PyGILState_Init(interp); @@ -2679,16 +2683,16 @@ new_interpreter(PyThreadState **tstate_p, } /* Copy the current interpreter config into the new interpreter */ - const PyConfig *src_config; + PyInterpreterState *src_interp; if (save_tstate != NULL) { - src_config = _PyInterpreterState_GetConfig(save_tstate->interp); + src_interp = save_tstate->interp; } else { /* No current thread state, copy from the main interpreter */ - PyInterpreterState *main_interp = _PyInterpreterState_Main(); - src_config = _PyInterpreterState_GetConfig(main_interp); + src_interp = _PyInterpreterState_Main(); } + const PyConfig *src_config = _PyInterpreterState_GetConfig(src_interp); /* This does not require that the GIL be held. */ status = _PyConfig_Copy(&interp->config, src_config); @@ -2696,6 +2700,10 @@ new_interpreter(PyThreadState **tstate_p, goto error; } + /* Copy this flag from source interpreter too. */ + interp->thread_inherit_context_warn = + src_interp->thread_inherit_context_warn; + /* This does not require that the GIL be held. */ status = init_interp_settings(interp, config); if (_PyStatus_EXCEPTION(status)) {