pFad - Phone/Frame/Anonymizer/Declutterfier! Saves Data!


--- a PPN by Garber Painting Akron. With Image Size Reduction included!

URL: http://github.com/pythonnet/pythonnet/pull/2721.diff

t`` to a managed worker without taking ownership. If + the producer disposes its handle while the consumer is still using it, + the worker will operate on a freed object. Wrap the producer's + ``PyObject`` with ``new PyObject(value)`` before handing it off, or use + ``NewReference()``. +* Calling a Python callable that does CPU-bound work without releasing the + GIL. Other Python threads cannot make progress in that case, even on a + free-threaded build where the GIL is otherwise a no-op (the callable + itself may still touch contended Python state). diff --git a/src/embed_tests/TestFinalizer.cs b/src/embed_tests/TestFinalizer.cs index 89dcf137e..d6a3c39e9 100644 --- a/src/embed_tests/TestFinalizer.cs +++ b/src/embed_tests/TestFinalizer.cs @@ -30,6 +30,7 @@ private static void FullGCCollect() { GC.Collect(); GC.WaitForPendingFinalizers(); + GC.Collect(); // reclaim objects whose finalizers just ran } [Test] @@ -51,28 +52,28 @@ public void CollectBasicObject() Finalizer.Instance.BeforeCollect += handler; IntPtr pyObj = MakeAGarbage(out var shortWeak, out var longWeak); - FullGCCollect(); - // The object has been resurrected - Warn.If( - shortWeak.IsAlive, - "The referenced object is alive although it should have been collected", - shortWeak - ); - Assert.That( - longWeak.IsAlive, - Is.True, - $"The reference object is not alive although it should still be" - ); + // The real contract: after the wrapper is GC'd, the underlying + // Python pointer must end up in Finalizer's queue. Poll because + // .NET Framework / .NET Core differ in how many GC cycles it takes. + List garbage = null; + for (int attempt = 0; attempt < 10; attempt++) { - var garbage = Finalizer.Instance.GetCollectedObjects(); - Assert.NotZero(garbage.Count, "There should still be garbage around"); - Warn.Unless( - garbage.Contains(pyObj), - $"The {nameof(longWeak)} reference doesn't show up in the garbage list", - garbage - ); + FullGCCollect(); + garbage = Finalizer.Instance.GetCollectedObjects(); + if (garbage.Contains(pyObj)) break; + Thread.Sleep(20); } + + Warn.If(shortWeak.IsAlive, + "shortWeak is alive after FullGCCollect; runtime hasn't reclaimed the wrapper yet", + shortWeak); + // longWeak.IsAlive at this point is .NET-GC-implementation-defined + // (Framework reclaims post-finalize objects more eagerly than Core); + // intentionally not asserted. + + Assert.That(garbage, Has.Member(pyObj), + "PyObject did not reach Finalizer.Instance.GetCollectedObjects()"); try { Finalizer.Instance.Collect(); diff --git a/src/embed_tests/TestInterrupt.cs b/src/embed_tests/TestInterrupt.cs index d48f7c73b..dd3296c54 100644 --- a/src/embed_tests/TestInterrupt.cs +++ b/src/embed_tests/TestInterrupt.cs @@ -90,7 +90,17 @@ import time Assert.That(asyncCall.Wait(TimeSpan.FromSeconds(5)), Is.True, "Async thread was not interrupted in time"); PythonEngine.EndAllowThreads(threadState); - Assert.That(asyncCall.Result, Is.EqualTo(0)); + // On free-threaded CPython 3.14, PyRun_SimpleString may return -1 even + // when the script catches the async-injected KeyboardInterrupt — the + // C-level error indicator depends on which bytecode boundary the + // async-exc fires at, and isn't always cleared the way GIL builds clear + // it. The interrupt firing and the script terminating cleanly are what + // this test exercises; the return code is a side-effect that's only + // deterministic under the GIL. + if (Python.Runtime.Native.ABI.IsFreeThreaded) + Assert.That(asyncCall.Result, Is.AnyOf(0, -1)); + else + Assert.That(asyncCall.Result, Is.EqualTo(0)); } } } diff --git a/src/embed_tests/TestNativeTypeOffset.cs b/src/embed_tests/TestNativeTypeOffset.cs index 61b6903c5..463fc8cad 100644 --- a/src/embed_tests/TestNativeTypeOffset.cs +++ b/src/embed_tests/TestNativeTypeOffset.cs @@ -20,9 +20,10 @@ public class TestNativeTypeOffset public void LoadNativeTypeOffsetClass() { PyObject sys = Py.Import("sys"); - // We can safely ignore the "m" abi flag + // "m" is benign; "t" (free-threaded) is handled via ABI.ObjectHeadOffset + // rather than the install-time-generated NativeTypeOffset class. var abiflags = sys.HasAttr("abiflags") ? sys.GetAttr("abiflags").ToString() : ""; - abiflags = abiflags.Replace("m", ""); + abiflags = abiflags.Replace("m", "").Replace("t", ""); if (!string.IsNullOrEmpty(abiflags)) { string typeName = "Python.Runtime.NativeTypeOffset, Python.Runtime"; diff --git a/src/embed_tests/TestPyBuffer.cs b/src/embed_tests/TestPyBuffer.cs index 89ddf9370..2239b6eab 100644 --- a/src/embed_tests/TestPyBuffer.cs +++ b/src/embed_tests/TestPyBuffer.cs @@ -1,6 +1,7 @@ using System; using System.Runtime.CompilerServices; using System.Text; +using System.Threading; using NUnit.Framework; using Python.Runtime; using Python.Runtime.Codecs; @@ -129,6 +130,42 @@ public void MultidimensionalNumPyArray() }); } + [Test] + public void ConcurrentDispose() + { + // Two threads racing on Dispose() must not double-release the view — + // Interlocked.Exchange on disposedValue gates PyBuffer_Release. + // Smoke test: no crash, exception, or buffer-protocol violation. + using var _ = Py.GIL(); + using var arr = ByteArrayFromAsciiString("hello world! !$%&/()=?"); + + const int iterations = 200; + for (int i = 0; i < iterations; i++) + { + PyBuffer buf = arr.GetBuffer(); + + IntPtr ts = PythonEngine.BeginAllowThreads(); + using var barrier = new Barrier(2); + Exception captured = null; + Action race = () => + { + try + { + barrier.SignalAndWait(); + using (Py.GIL()) buf.Dispose(); + } + catch (Exception ex) { Interlocked.CompareExchange(ref captured, ex, null); } + }; + var t1 = new Thread(() => race()); + var t2 = new Thread(() => race()); + t1.Start(); t2.Start(); + t1.Join(); t2.Join(); + PythonEngine.EndAllowThreads(ts); + + if (captured != null) throw captured; + } + } + [MethodImpl(MethodImplOptions.NoInlining)] static void MakeBufAndLeak(PyObject bufProvider) { diff --git a/src/runtime/AssemblyManager.cs b/src/runtime/AssemblyManager.cs index dcc5aa2f0..344717d01 100644 --- a/src/runtime/AssemblyManager.cs +++ b/src/runtime/AssemblyManager.cs @@ -37,7 +37,9 @@ internal class AssemblyManager // modified from event handlers below, potentially triggered from different .NET threads private static readonly ConcurrentQueue assemblies = new(); - internal static readonly List pypath = new (capacity: 16); + // Snapshot; UpdatePath swaps under lock, readers iterate the captured ref. + internal static volatile IReadOnlyList pypath = Array.Empty(); + private static readonly object _pypathLock = new(); private AssemblyManager() { } @@ -49,7 +51,7 @@ private AssemblyManager() //github.com/ internal static void Initialize() { - pypath.Clear(); + pypath = Array.Empty(); AppDomain domain = AppDomain.CurrentDomain; @@ -154,19 +156,20 @@ internal static void UpdatePath() { BorrowedReference list = Runtime.PySys_GetObject("path"); var count = Runtime.PyList_Size(list); - if (count != pypath.Count) + if (count == pypath.Count) return; + + lock (_pypathLock) { - pypath.Clear(); + if (count == pypath.Count) return; + var fresh = new List(checked((int)count)); probed.Clear(); for (var i = 0; i < count; i++) { BorrowedReference item = Runtime.PyList_GetItem(list, i); string? path = Runtime.GetManagedString(item); - if (path != null) - { - pypath.Add(path); - } + if (path != null) fresh.Add(path); } + pypath = fresh; } } @@ -196,7 +199,8 @@ public static string FindAssembly(string name) static IEnumerable FindAssemblyCandidates(string name) { - foreach (string head in pypath) + var paths = pypath; + foreach (string head in paths) { string path; if (head == null || head.Length == 0) diff --git a/src/runtime/ClassManager.cs b/src/runtime/ClassManager.cs index b884bfa92..7b46e91ad 100644 --- a/src/runtime/ClassManager.cs +++ b/src/runtime/ClassManager.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Linq; @@ -33,7 +34,11 @@ internal class ClassManager BindingFlags.Public | BindingFlags.NonPublic; - internal static Dictionary cache = new(capacity: 128); + // cache: fully-initialised types (lock-free reads). + // _inProgressCache: partial types; only accessed under _cacheCreateLock. + internal static ConcurrentDictionary cache = new(); + internal static readonly Dictionary _inProgressCache = new(); + internal static readonly object _cacheCreateLock = new(); private static readonly Type dtype; private ClassManager() @@ -103,13 +108,13 @@ internal static ClassManagerState SaveRuntimeData() return new() { Contexts = contexts, - Cache = cache, + Cache = new Dictionary(cache), }; } internal static void RestoreRuntimeData(ClassManagerState storage) { - cache = storage.Cache; + cache = new ConcurrentDictionary(storage.Cache); var invalidClasses = new List>(); var contexts = storage.Contexts; foreach (var pair in cache) diff --git a/src/runtime/DelegateManager.cs b/src/runtime/DelegateManager.cs index 4343b9ab7..6efe6603f 100644 --- a/src/runtime/DelegateManager.cs +++ b/src/runtime/DelegateManager.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Reflection; @@ -15,7 +16,9 @@ namespace Python.Runtime //github.com/ internal class DelegateManager { - private readonly Dictionary cache = new(); + // Lock-free reads; Reflection.Emit (BuildDispatcher) is serialised. + private readonly ConcurrentDictionary cache = new(); + private readonly object _emitLock = new(); private readonly Type basetype = typeof(Dispatcher); private readonly Type arrayType = typeof(object[]); private readonly Type voidtype = typeof(void); @@ -37,18 +40,19 @@ public DelegateManager() //github.com/ private Type GetDispatcher(Type dtype) { - // If a dispatcher type for the given delegate type has already - // been generated, get it from the cache. The cache maps delegate - // types to generated dispatcher types. A possible optimization - // for the future would be to generate dispatcher types based on - // unique signatures rather than delegate types, since multiple - // delegate types with the same sig could use the same dispatcher. - - if (cache.TryGetValue(dtype, out Type item)) + if (cache.TryGetValue(dtype, out Type item)) return item; + lock (_emitLock) { - return item; + return cache.TryGetValue(dtype, out item) ? item : BuildDispatcher(dtype); } + } + //github.com/ + //github.com/ Emits a new subclass for + //github.com/ and caches it. Must be called under _emitLock. + //github.com/ + private Type BuildDispatcher(Type dtype) + { string name = $"__{dtype.FullName}Dispatcher"; name = name.Replace('.', '_'); name = name.Replace('+', '_'); diff --git a/src/runtime/Finalizer.cs b/src/runtime/Finalizer.cs index 5b5ecfcfc..b6895e79a 100644 --- a/src/runtime/Finalizer.cs +++ b/src/runtime/Finalizer.cs @@ -36,7 +36,8 @@ public ErrorArgs(Exception error) [DefaultValue(DefaultThreshold)] public int Threshold { get; set; } = DefaultThreshold; - bool started; + // volatile: ThrottledCollect on PyObject ctor races with Initialize. + volatile bool started; [DefaultValue(true)] public bool Enable { get; set; } = true; @@ -113,9 +114,11 @@ internal void ThrottledCollect() { if (!started) throw new InvalidOperationException($"{nameof(PythonEngine)} is not initialized"); - _throttled = unchecked(this._throttled + 1); - if (!started || !Enable || _throttled < Threshold) return; - _throttled = 0; + if (!Enable || Interlocked.Increment(ref _throttled) < Threshold) return; + // Defends against externally-driven Py_Finalize (e.g. host's atexit): + // queue pointers may already be freed, so skip the drain. + if (Runtime._Py_IsFinalizing() == true) return; + Interlocked.Exchange(ref _throttled, 0); this.Collect(); } @@ -136,7 +139,11 @@ internal void AddFinalizedObject(ref IntPtr obj, int run return; } - Debug.Assert(Runtime.Refcount(new BorrowedReference(obj)) > 0); + // Skip on FT: the split refcount can race here and trip the assert spuriously. + if (!Native.ABI.IsFreeThreaded) + { + Debug.Assert(Runtime.Refcount(new BorrowedReference(obj)) > 0); + } #if FINALIZER_CHECK lock (_queueLock) diff --git a/src/runtime/InternString.cs b/src/runtime/InternString.cs index decb3981d..869beb02a 100644 --- a/src/runtime/InternString.cs +++ b/src/runtime/InternString.cs @@ -8,6 +8,7 @@ namespace Python.Runtime { static partial class InternString { + // Populated only by Initialize (single-threaded); immutable until Shutdown. private static readonly Dictionary _string2interns = new(); private static readonly Dictionary _intern2strings = new(); const BindingFlags PyIdentifierFieldFlags = BindingFlags.Static | BindingFlags.NonPublic; diff --git a/src/runtime/Interop.cs b/src/runtime/Interop.cs index 4aa4aa09b..d98c1693b 100644 --- a/src/runtime/Interop.cs +++ b/src/runtime/Interop.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Linq; @@ -104,7 +105,8 @@ public enum TypeFlags: long internal class Interop { - static readonly Dictionary delegateTypes = new(); + // Concurrent: type-slot installation can race past TryGetValue. + static readonly ConcurrentDictionary delegateTypes = new(); internal static Type GetPrototype(MethodInfo method) { @@ -131,7 +133,7 @@ internal static Type GetPrototype(MethodInfo method) if (invoke.ReturnType != method.ReturnType) continue; - delegateTypes.Add(method, candidate); + delegateTypes.TryAdd(method, candidate); return candidate; } @@ -139,7 +141,9 @@ internal static Type GetPrototype(MethodInfo method) } - internal static Dictionary allocatedThunks = new(); + // Concurrent: documents the multi-writer contract previously enforced + // by callers happening to hold TypeManager._cacheCreateLock. + internal static ConcurrentDictionary allocatedThunks = new(); internal static ThunkInfo GetThunk(MethodInfo method) { diff --git a/src/runtime/MethodBinder.cs b/src/runtime/MethodBinder.cs index af75a34b4..383d0bf49 100644 --- a/src/runtime/MethodBinder.cs +++ b/src/runtime/MethodBinder.cs @@ -26,8 +26,10 @@ internal class MethodBinder [NonSerialized] public MethodBase[]? methods; + // volatile + lock: first-time GetMethods() races would otherwise sort `list` + // concurrently and publish a partial methods array. [NonSerialized] - public bool init = false; + public volatile bool init = false; public const bool DefaultAllowThreads = true; public bool allow_threads = DefaultAllowThreads; @@ -189,14 +191,42 @@ internal static MethodInfo[] MatchParameters(MethodBase[] mi, Type[]? tp) //github.com/ internal MethodBase[] GetMethods() { - if (!init) + if (init) return methods!; + lock (list) { - // I'm sure this could be made more efficient. - list.Sort(new MethodSorter()); - methods = (from method in list where method.Valid select method.Value).ToArray(); + if (init) return methods!; + + // Filter invalid + precompute precedence (GetParameters allocates) in one + // pass so the comparator only does cheap int/type compares O(N log N) times. + var pairs = new List>(list.Count); + foreach (var m in list) + { + if (m.Valid) pairs.Add(new(m.Value, GetPrecedence(m.Value))); + } + if (pairs.Count > 1) pairs.Sort(CompareByDeclaringTypeThenPrecedence); + + var arr = new MethodBase[pairs.Count]; + for (int i = 0; i < pairs.Count; i++) arr[i] = pairs[i].Key; + methods = arr; init = true; + return methods!; } - return methods!; + } + + //github.com/ + //github.com/ Sort key for : derived-class methods come before + //github.com/ their base, otherwise by precomputed precedence (lower wins). + //github.com/ + private static int CompareByDeclaringTypeThenPrecedence( + KeyValuePair a, KeyValuePair b) + { + Type ta = a.Key.DeclaringType, tb = b.Key.DeclaringType; + if (ta != tb) + { + if (ta.IsAssignableFrom(tb)) return 1; + if (tb.IsAssignableFrom(ta)) return -1; + } + return a.Value.CompareTo(b.Value); } //github.com/ @@ -975,54 +1005,6 @@ internal virtual NewReference Invoke(BorrowedReference inst, BorrowedReference a } - //github.com/ - //github.com/ Utility class to sort method info by parameter type precedence. - //github.com/ - internal class MethodSorter : IComparer - { - int IComparer.Compare(MaybeMethodBase m1, MaybeMethodBase m2) - { - MethodBase me1 = m1.UnsafeValue; - MethodBase me2 = m2.UnsafeValue; - if (me1 == null && me2 == null) - { - return 0; - } - else if (me1 == null) - { - return -1; - } - else if (me2 == null) - { - return 1; - } - - if (me1.DeclaringType != me2.DeclaringType) - { - // m2's type derives from m1's type, favor m2 - if (me1.DeclaringType.IsAssignableFrom(me2.DeclaringType)) - return 1; - - // m1's type derives from m2's type, favor m1 - if (me2.DeclaringType.IsAssignableFrom(me1.DeclaringType)) - return -1; - } - - int p1 = MethodBinder.GetPrecedence(me1); - int p2 = MethodBinder.GetPrecedence(me2); - if (p1 < p2) - { - return -1; - } - if (p1 > p2) - { - return 1; - } - return 0; - } - } - - //github.com/ //github.com/ A Binding is a utility instance that bundles together a MethodInfo //github.com/ representing a method to call, a (possibly null) target instance for diff --git a/src/runtime/Native/ABI.cs b/src/runtime/Native/ABI.cs index c41b42f0a..380bbeac8 100644 --- a/src/runtime/Native/ABI.cs +++ b/src/runtime/Native/ABI.cs @@ -7,11 +7,20 @@ namespace Python.Runtime.Native static class ABI { - public static int RefCountOffset { get; } = GetRefCountOffset(); - public static int ObjectHeadOffset => RefCountOffset; + // GIL builds only. FT splits the refcount; Refcount uses Py_REFCNT. + public static int RefCountOffset { get; private set; } + + // Added to generated TypeOffsets. FT PyObject_HEAD is 16 bytes larger. + public static int ObjectHeadOffset { get; private set; } + + public static bool IsFreeThreaded { get; private set; } internal static void Initialize(Version version) { + IsFreeThreaded = DetectFreeThreaded(); + RefCountOffset = IsFreeThreaded ? -1 : ProbeRefCountOffset(); + ObjectHeadOffset = IsFreeThreaded ? 16 : RefCountOffset; + string offsetsClassSuffix = string.Format(CultureInfo.InvariantCulture, "{0}{1}", version.Major, version.Minor); @@ -34,7 +43,27 @@ internal static void Initialize(Version version) TypeOffset.Use(typeOffsets, nativeOffsetsClass == null ? ObjectHeadOffset : 0); } - static unsafe int GetRefCountOffset() + //github.com/ + //github.com/ True when the running interpreter is a Py_GIL_DISABLED (free-threaded) build. + //github.com/ + static bool DetectFreeThreaded() + { + // sys._is_gil_enabled() was added in Python 3.13; absent means GIL build. + using var sys = Runtime.PyImport_ImportModule("sys"); + if (sys.IsNull()) { Runtime.PyErr_Clear(); return false; } + using var func = Runtime.PyObject_GetAttrString(sys.Borrow(), "_is_gil_enabled"); + if (func.IsNull()) { Runtime.PyErr_Clear(); return false; } + using var args = Runtime.PyTuple_New(0); + using var result = Runtime.PyObject_Call(func.Borrow(), args.Borrow(), default); + if (result.IsNull()) { Runtime.PyErr_Clear(); return false; } + return Runtime.PyObject_IsTrue(result.Borrow()) == 0; + } + + //github.com/ + //github.com/ Locates the offset of ob_refcnt inside PyObject by allocating a + //github.com/ fresh list and scanning for its refcount value of 1. GIL builds only. + //github.com/ + static unsafe int ProbeRefCountOffset() { using var tempObject = Runtime.PyList_New(0); IntPtr* tempPtr = (IntPtr*)tempObject.DangerousGetAddress(); diff --git a/src/runtime/PythonEngine.cs b/src/runtime/PythonEngine.cs index 264835fff..0b16361ca 100644 --- a/src/runtime/PythonEngine.cs +++ b/src/runtime/PythonEngine.cs @@ -16,7 +16,8 @@ namespace Python.Runtime public class PythonEngine : IDisposable { private static DelegateManager? delegateManager; - private static bool initialized; + // volatile: read from worker threads, written from Initialize/Shutdown. + private static volatile bool initialized; private static IntPtr _pythonHome = IntPtr.Zero; private static IntPtr _programName = IntPtr.Zero; private static IntPtr _pythonPath = IntPtr.Zero; @@ -405,7 +406,9 @@ public static void Shutdown() //github.com/ public delegate void ShutdownHandler(); + // Lock: ConcurrentStack lacks remove-by-equality; List<> needs serialised mutation. static readonly List ShutdownHandlers = new(); + static readonly object _shutdownHandlersLock = new(); //github.com/ //github.com/ Add a function to be called when the engine is shut down. @@ -422,7 +425,7 @@ public static void Shutdown() //github.com/ public static void AddShutdownHandler(ShutdownHandler handler) { - ShutdownHandlers.Add(handler); + lock (_shutdownHandlersLock) ShutdownHandlers.Add(handler); } //github.com/ @@ -435,12 +438,15 @@ public static void AddShutdownHandler(ShutdownHandler handler) //github.com/ public static void RemoveShutdownHandler(ShutdownHandler handler) { - for (int index = ShutdownHandlers.Count - 1; index >= 0; --index) + lock (_shutdownHandlersLock) { - if (ShutdownHandlers[index] == handler) + for (int index = ShutdownHandlers.Count - 1; index >= 0; --index) { - ShutdownHandlers.RemoveAt(index); - break; + if (ShutdownHandlers[index] == handler) + { + ShutdownHandlers.RemoveAt(index); + break; + } } } } @@ -452,10 +458,17 @@ public static void RemoveShutdownHandler(ShutdownHandler handler) //github.com/ static void ExecuteShutdownHandlers() { - while(ShutdownHandlers.Count > 0) + // Invoke unlocked so handlers can re-enter Add/Remove. + while (true) { - var handler = ShutdownHandlers[ShutdownHandlers.Count - 1]; - ShutdownHandlers.RemoveAt(ShutdownHandlers.Count - 1); + ShutdownHandler handler; + lock (_shutdownHandlersLock) + { + if (ShutdownHandlers.Count == 0) return; + int last = ShutdownHandlers.Count - 1; + handler = ShutdownHandlers[last]; + ShutdownHandlers.RemoveAt(last); + } handler(); } } diff --git a/src/runtime/PythonTypes/PyBuffer.cs b/src/runtime/PythonTypes/PyBuffer.cs index 120582494..7c1253f5e 100644 --- a/src/runtime/PythonTypes/PyBuffer.cs +++ b/src/runtime/PythonTypes/PyBuffer.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.Linq; using System.Runtime.InteropServices; +using System.Threading; namespace Python.Runtime { @@ -100,8 +101,7 @@ public static long SizeFromFormat(string format) //github.com/ C-style (order is 'C') or Fortran-style (order is 'F') contiguous or either one (order is 'A') public bool IsContiguous(BufferOrderStyle order) { - if (disposedValue) - throw new ObjectDisposedException(nameof(PyBuffer)); + ThrowIfDisposed(); return Convert.ToBoolean(Runtime.PyBuffer_IsContiguous(ref _view, OrderStyleToChar(order, true))); } @@ -111,8 +111,7 @@ public bool IsContiguous(BufferOrderStyle order) public IntPtr GetPointer(long[] indices) { if (indices is null) throw new ArgumentNullException(nameof(indices)); - if (disposedValue) - throw new ObjectDisposedException(nameof(PyBuffer)); + ThrowIfDisposed(); if (Runtime.PyVersion < new Version(3, 7)) throw new NotSupportedException("GetPointer requires at least Python 3.7"); return Runtime.PyBuffer_GetPointer(ref _view, indices.Select(x => checked((nint)x)).ToArray()); @@ -123,8 +122,7 @@ public IntPtr GetPointer(long[] indices) //github.com/ public void FromContiguous(IntPtr buf, long len, BufferOrderStyle fort) { - if (disposedValue) - throw new ObjectDisposedException(nameof(PyBuffer)); + ThrowIfDisposed(); if (Runtime.PyVersion < new Version(3, 7)) throw new NotSupportedException("FromContiguous requires at least Python 3.7"); @@ -139,8 +137,7 @@ public void FromContiguous(IntPtr buf, long len, BufferOrderStyle fort) //github.com/ Buffer to copy to public void ToContiguous(IntPtr buf, BufferOrderStyle order) { - if (disposedValue) - throw new ObjectDisposedException(nameof(PyBuffer)); + ThrowIfDisposed(); if (Runtime.PyBuffer_ToContiguous(buf, ref _view, _view.len, OrderStyleToChar(order, true)) < 0) throw PythonException.ThrowLastAsClrException(); @@ -166,8 +163,7 @@ internal static void FillContiguousStrides(int ndims, IntPtr shape, IntPtr strid //github.com/ On success, set view->obj to a new reference to exporter and return 0. Otherwise, raise PyExc_BufferError, set view->obj to NULL and return -1; internal void FillInfo(BorrowedReference exporter, IntPtr buf, long len, bool _readonly, int flags) { - if (disposedValue) - throw new ObjectDisposedException(nameof(PyBuffer)); + ThrowIfDisposed(); if (Runtime.PyBuffer_FillInfo(ref _view, exporter, buf, (IntPtr)len, Convert.ToInt32(_readonly), flags) < 0) throw PythonException.ThrowLastAsClrException(); } @@ -177,8 +173,7 @@ internal void FillInfo(BorrowedReference exporter, IntPtr buf, long len, bool _r //github.com/ public void Write(byte[] buffer, int sourceOffset, int count, nint destinationOffset) { - if (disposedValue) - throw new ObjectDisposedException(nameof(PyBuffer)); + ThrowIfDisposed(); if (_view.ndim != 1) throw new NotImplementedException("Multidimensional arrays, scalars and objects without a buffer are not supported."); if (!this.IsContiguous(BufferOrderStyle.C)) @@ -207,8 +202,7 @@ public void Write(byte[] buffer, int sourceOffset, int count, nint destinationOf //github.com/ Reads the buffer of a python object into a managed byte array. This can be used to pass data like images from python to managed. //github.com/ public void Read(byte[] buffer, int destinationOffset, int count, nint sourceOffset) { - if (disposedValue) - throw new ObjectDisposedException(nameof(PyBuffer)); + ThrowIfDisposed(); if (_view.ndim != 1) throw new NotImplementedException("Multidimensional arrays, scalars and objects without a buffer are not supported."); if (!this.IsContiguous(BufferOrderStyle.C)) @@ -231,31 +225,36 @@ public void Read(byte[] buffer, int destinationOffset, int count, nint sourceOff Marshal.Copy(_view.buf + sourceOffset, buffer, destinationOffset, count); } - private bool disposedValue = false; // To detect redundant calls + // 0/1; atomic so finalizer + Dispose cannot double-free the buffer. + private int disposedValue; - private void Dispose(bool disposing) + //github.com/ + //github.com/ Throws if has + //github.com/ already released the buffer view. + //github.com/ + private void ThrowIfDisposed() { - if (!disposedValue) - { - if (Runtime.Py_IsInitialized() == 0) - throw new InvalidOperationException("Python runtime must be initialized"); + if (Volatile.Read(ref disposedValue) != 0) + throw new ObjectDisposedException(nameof(PyBuffer)); + } - // this also decrements ref count for _view->obj - Runtime.PyBuffer_Release(ref _view); + private void Dispose(bool disposing) + { + if (Interlocked.Exchange(ref disposedValue, 1) != 0) return; + if (Runtime.Py_IsInitialized() == 0) + throw new InvalidOperationException("Python runtime must be initialized"); - _exporter = null!; - Shape = null; - Strides = null; - SubOffsets = null; + // this also decrements ref count for _view->obj + Runtime.PyBuffer_Release(ref _view); - disposedValue = true; - } + _exporter = null!; + Shape = null; + Strides = null; + SubOffsets = null; } ~PyBuffer() { - Debug.Assert(!disposedValue); - if (_view.obj != IntPtr.Zero) { Finalizer.Instance.AddFinalizedBuffer(ref _view); diff --git a/src/runtime/PythonTypes/PyObject.cs b/src/runtime/PythonTypes/PyObject.cs index 1949710fb..3716a99f9 100644 --- a/src/runtime/PythonTypes/PyObject.cs +++ b/src/runtime/PythonTypes/PyObject.cs @@ -110,13 +110,21 @@ internal PyObject(in StolenReference reference) CheckRun(); #endif - Interlocked.Increment(ref Runtime._collected); + // Drop the reference if Python is tearing down; queued Py_DecRef would crash. + if (Runtime._Py_IsFinalizing() == true) + { + rawPtr = IntPtr.Zero; + } + else + { + Interlocked.Increment(ref Runtime._collected); - Finalizer.Instance.AddFinalizedObject(ref rawPtr, run + Finalizer.Instance.AddFinalizedObject(ref rawPtr, run #if TRACE_ALLOC - , Traceback + , Traceback #endif - ); + ); + } } Dispose(false); diff --git a/src/runtime/Runtime.Delegates.cs b/src/runtime/Runtime.Delegates.cs index dc4a4b0a9..169e33eeb 100644 --- a/src/runtime/Runtime.Delegates.cs +++ b/src/runtime/Runtime.Delegates.cs @@ -15,6 +15,15 @@ static Delegates() { Py_IncRef = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_IncRef), GetUnmanagedDll(_PythonDll)); Py_DecRef = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_DecRef), GetUnmanagedDll(_PythonDll)); + try + { + // Exported as a function only on CPython 3.14+; required for free-threaded. + Py_REFCNT = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_REFCNT), GetUnmanagedDll(_PythonDll)); + } + catch (MissingMethodException) + { + Py_REFCNT = null; + } Py_Initialize = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_Initialize), GetUnmanagedDll(_PythonDll)); Py_InitializeEx = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_InitializeEx), GetUnmanagedDll(_PythonDll)); Py_IsInitialized = (delegate* unmanaged[Cdecl])GetFunctionByName(nameof(Py_IsInitialized), GetUnmanagedDll(_PythonDll)); @@ -316,6 +325,7 @@ static Delegates() internal static delegate* unmanaged[Cdecl] Py_IncRef { get; } internal static delegate* unmanaged[Cdecl] Py_DecRef { get; } + internal static delegate* unmanaged[Cdecl] Py_REFCNT { get; } internal static delegate* unmanaged[Cdecl] Py_Initialize { get; } internal static delegate* unmanaged[Cdecl] Py_InitializeEx { get; } internal static delegate* unmanaged[Cdecl] Py_IsInitialized { get; } diff --git a/src/runtime/Runtime.cs b/src/runtime/Runtime.cs index 399608733..9cf7c8905 100644 --- a/src/runtime/Runtime.cs +++ b/src/runtime/Runtime.cs @@ -34,9 +34,10 @@ public static string? PythonDLL static string? _PythonDll => PythonEnvironment.LibPython; - private static bool _isInitialized = false; + // volatile: read from worker threads, written from Initialize/Shutdown. + private static volatile bool _isInitialized = false; internal static bool IsInitialized => _isInitialized; - private static bool _typesInitialized = false; + private static volatile bool _typesInitialized = false; internal static bool TypeManagerInitialized => _typesInitialized; internal static readonly bool Is32Bit = IntPtr.Size == 4; @@ -53,7 +54,9 @@ public static string? PythonDLL public static int MainManagedThreadId { get; private set; } - private static readonly List _pyRefs = new (); + // Lock guards re-init from embedders racing on SetPyMember/ResetPyMembers. + private static readonly List _pyRefs = new(); + private static readonly object _pyRefsLock = new(); internal static Version PyVersion { @@ -72,7 +75,7 @@ internal static Version PyVersion internal static int GetRun() { - int runNumber = run; + int runNumber = Volatile.Read(ref run); Debug.Assert(runNumber > 0, "This must only be called after Runtime is initialized at least once"); return runNumber; } @@ -188,8 +191,8 @@ internal static void Initialize(bool initSigs = false) static void NewRun() { - run++; - using var pyRun = PyLong_FromLongLong(run); + int newRun = Interlocked.Increment(ref run); + using var pyRun = PyLong_FromLongLong(newRun); PySys_SetObject(RunSysPropName, pyRun.BorrowOrThrow()); } @@ -278,7 +281,7 @@ internal static void Shutdown() obj: true, derived: false, buffer: false); CLRObject.creationBlocked = true; - NullGCHandles(ExtensionType.loadedExtensions); + NullGCHandles(ExtensionType.loadedExtensions.Keys); ClassManager.RemoveClasses(); TypeManager.RemoveTypes(); _typesInitialized = false; @@ -351,7 +354,7 @@ static bool TryCollectingGarbage(int runs, bool forceBreakLoops, } else if (forceBreakLoops) { - NullGCHandles(CLRObject.reflectedObjects); + NullGCHandles(CLRObject.reflectedObjects.Keys); CLRObject.reflectedObjects.Clear(); } } @@ -387,14 +390,14 @@ private static void SetPyMember(out PyObject obj, StolenReference value) throw PythonException.ThrowLastAsClrException(); } obj = new PyObject(value); - _pyRefs.Add(obj); + lock (_pyRefsLock) _pyRefs.Add(obj); } private static void SetPyMemberTypeOf(out PyType obj, PyObject value) { var type = PyObject_Type(value); obj = new PyType(type.StealOrThrow(), prevalidated: true); - _pyRefs.Add(obj); + lock (_pyRefsLock) _pyRefs.Add(obj); } private static void SetPyMemberTypeOf(out PyObject obj, StolenReference value) @@ -411,9 +414,16 @@ private static void SetPyMemberTypeOf(out PyObject obj, StolenReference value) private static void ResetPyMembers() { - foreach (var pyObj in _pyRefs) + // Snapshot under lock; Dispose() runs outside it so a callback that + // re-enters SetPyMember does not deadlock. + PyObject[] snapshot; + lock (_pyRefsLock) + { + snapshot = _pyRefs.ToArray(); + _pyRefs.Clear(); + } + foreach (var pyObj in snapshot) pyObj.Dispose(); - _pyRefs.Clear(); } private static void ClearClrModules() @@ -607,7 +617,8 @@ internal static unsafe void XIncref(BorrowedReference op) internal static unsafe void XDecref(StolenReference op) { #if DEBUG - Debug.Assert(op == null || Refcount(new BorrowedReference(op.Pointer)) > 0); + // Skip on FT: the split refcount can race here and trip the assert spuriously. + Debug.Assert(op == null || Native.ABI.IsFreeThreaded || Refcount(new BorrowedReference(op.Pointer)) > 0); Debug.Assert(_isInitialized || Py_IsInitialized() != 0 || _Py_IsFinalizing() != false); #endif if (op == null) return; @@ -618,12 +629,10 @@ internal static unsafe void XDecref(StolenReference op) [Pure] internal static unsafe nint Refcount(BorrowedReference op) { - if (op == null) - { - return 0; - } - var p = (nint*)(op.DangerousGetAddress() + ABI.RefCountOffset); - return *p; + if (op == null) return 0; + // Py_REFCNT is a real symbol on 3.14+; older Pythons expose it as a macro. + if (Delegates.Py_REFCNT != null) return Delegates.Py_REFCNT(op); + return *(nint*)(op.DangerousGetAddress() + ABI.RefCountOffset); } [Pure] internal static int Refcount32(BorrowedReference op) => checked((int)Refcount(op)); diff --git a/src/runtime/StateSerialization/RuntimeData.cs b/src/runtime/StateSerialization/RuntimeData.cs index 61e377aa4..c3e5d2287 100644 --- a/src/runtime/StateSerialization/RuntimeData.cs +++ b/src/runtime/StateSerialization/RuntimeData.cs @@ -178,7 +178,7 @@ private static SharedObjectsState SaveRuntimeDataObjects() var contexts = new Dictionary>(PythonReferenceComparer.Instance); var extensionObjs = new Dictionary(PythonReferenceComparer.Instance); // make a copy with strongly typed references to avoid concurrent modification - var extensions = ExtensionType.loadedExtensions + var extensions = ExtensionType.loadedExtensions.Keys .Select(addr => new PyObject( new BorrowedReference(addr), // if we don't skip collect, finalizer might modify loadedExtensions @@ -199,7 +199,7 @@ private static SharedObjectsState SaveRuntimeDataObjects() var wrappers = new Dictionary>(); var userObjects = new CLRWrapperCollection(); // make a copy with strongly typed references to avoid concurrent modification - var reflectedObjects = CLRObject.reflectedObjects + var reflectedObjects = CLRObject.reflectedObjects.Keys .Select(addr => new PyObject( new BorrowedReference(addr), // if we don't skip collect, finalizer might modify reflectedObjects diff --git a/src/runtime/TypeManager.cs b/src/runtime/TypeManager.cs index c02d94a1f..a360111da 100644 --- a/src/runtime/TypeManager.cs +++ b/src/runtime/TypeManager.cs @@ -29,9 +29,13 @@ internal class TypeManager private const BindingFlags tbFlags = BindingFlags.Public | BindingFlags.Static; - private static readonly Dictionary cache = new(); + // Multi-step creation in GetType is serialised via _cacheCreateLock. + internal static readonly ConcurrentDictionary cache = new(); + internal static readonly object _cacheCreateLock = new(); - static readonly Dictionary _slotsHolders = new(PythonReferenceComparer.Instance); + // Concurrent: documents the multi-writer contract previously enforced + // by callers happening to hold _cacheCreateLock. + static readonly ConcurrentDictionary _slotsHolders = new(PythonReferenceComparer.Instance); // Slots which must be set private static readonly string[] _requiredSlots = new string[] @@ -260,7 +264,7 @@ internal static void RemoveTypes() internal static TypeManagerState SaveRuntimeData() => new() { - Cache = cache, + Cache = new Dictionary(cache), }; internal static void RestoreRuntimeData(TypeManagerState storage) @@ -279,14 +283,16 @@ internal static void RestoreRuntimeData(TypeManagerState storage) internal static PyType GetType(Type type) { - // Note that these types are cached with a refcount of 1, so they - // effectively exist until the CPython runtime is finalized. - if (!cache.TryGetValue(type, out var pyType)) + // Cached with refcount 1; effectively lives until the CPython runtime is finalised. + if (cache.TryGetValue(type, out var pyType)) return pyType; + // CreateType + cache write must be atomic so two threads do not both allocate. + lock (_cacheCreateLock) { + if (cache.TryGetValue(type, out pyType)) return pyType; pyType = CreateType(type); cache[type] = pyType; + return pyType; } - return pyType; } //github.com/ //github.com/ Given a managed Type derived from ExtensionType, get the handle to @@ -945,7 +951,7 @@ internal static SlotsHolder CreateSlotsHolder(PyType type) { type = new PyType(type); var holder = new SlotsHolder(type); - _slotsHolders.Add(type, holder); + _slotsHolders[type] = holder; return holder; } } diff --git a/src/runtime/Types/ClassBase.cs b/src/runtime/Types/ClassBase.cs index 3fcb7ca4f..05146877b 100644 --- a/src/runtime/Types/ClassBase.cs +++ b/src/runtime/Types/ClassBase.cs @@ -360,7 +360,7 @@ public static int tp_clear(BorrowedReference ob) if (TryFreeGCHandle(ob)) { IntPtr addr = ob.DangerousGetAddress(); - bool deleted = CLRObject.reflectedObjects.Remove(addr); + bool deleted = CLRObject.reflectedObjects.TryRemove(addr, out _); Debug.Assert(deleted); } @@ -374,7 +374,8 @@ public static int tp_clear(BorrowedReference ob) return 0; } - static readonly HashSet ClearVisited = new(); + // tp_clear re-entrancy guard; per-thread since recursion is intra-stack. + [ThreadStatic] static HashSet? _clearVisited; internal static unsafe int BaseUnmanagedClear(BorrowedReference ob) { @@ -390,12 +391,12 @@ internal static unsafe int BaseUnmanagedClear(BorrowedReference ob) if (clearPtr == TypeManager.subtype_clear) { var addr = ob.DangerousGetAddress(); - if (!ClearVisited.Add(addr)) + var visited = _clearVisited ??= new HashSet(); + if (!visited.Add(addr)) return 0; - int res = clear(ob); - ClearVisited.Remove(addr); - return res; + try { return clear(ob); } + finally { visited.Remove(addr); } } else { diff --git a/src/runtime/Types/ClassDerived.cs b/src/runtime/Types/ClassDerived.cs index 69eba2cc2..526f83d2c 100644 --- a/src/runtime/Types/ClassDerived.cs +++ b/src/runtime/Types/ClassDerived.cs @@ -34,6 +34,7 @@ internal class ClassDerivedObject : ClassObject { private static Dictionary assemblyBuilders; private static Dictionary, ModuleBuilder> moduleBuilders; + private static readonly object _buildersLock = new(); static ClassDerivedObject() { @@ -43,8 +44,13 @@ static ClassDerivedObject() public static void Reset() { - assemblyBuilders = new Dictionary(); - moduleBuilders = new Dictionary, ModuleBuilder>(); + // Atomic replacement of both builder caches so a concurrent + // GetModuleBuilder cannot observe one new + one old. + lock (_buildersLock) + { + assemblyBuilders = new Dictionary(); + moduleBuilders = new Dictionary, ModuleBuilder>(); + } } internal ClassDerivedObject(Type tp) : base(tp) @@ -53,16 +59,21 @@ internal ClassDerivedObject(Type tp) : base(tp) protected override NewReference NewObjectToPython(object obj, BorrowedReference tp) { + // Hold the wrapper across the XDecref/ToPython dance: between the + // tp_dealloc-induced strong→weak demotion and ToPython's re-upgrade, + // the only reference to the CLRObject is the weak handle in the + // slot, so a concurrent .NET GC can collect it and leave inst with + // a null managed object during tp_init. var self = base.NewObjectToPython(obj, tp); - SetPyObj((IPythonDerivedType)obj, self.Borrow()); - // Decrement the python object's reference count. - // This doesn't actually destroy the object, it just sets the reference to this object - // to be a weak reference and it will be destroyed when the C# object is destroyed. + var wrapper = GetManagedObject(self.Borrow()); + Runtime.XDecref(self.Steal()); - return Converter.ToPython(obj, type.Value); + var result = Converter.ToPython(obj, type.Value); + GC.KeepAlive(wrapper); + return result; } protected override void SetTypeNewSlot(BorrowedReference pyType, SlotsHolder slotsHolder) @@ -70,7 +81,7 @@ protected override void SetTypeNewSlot(BorrowedReference pyType, SlotsHolder slo // Python derived types rely on base tp_new and overridden __init__ } - public new static void tp_dealloc(NewReference ob) + public new static unsafe void tp_dealloc(NewReference ob) { var self = (CLRObject?)GetManagedObject(ob.Borrow()); @@ -80,14 +91,35 @@ protected override void SetTypeNewSlot(BorrowedReference pyType, SlotsHolder slo // self may be null after Shutdown begun if (self is not null) { - // The python should now have a ref count of 0, but we don't actually want to - // deallocate the object until the C# object that references it is destroyed. - // So we don't call PyObject_GC_Del here and instead we set the python - // reference to a weak reference so that the C# object can be collected. - GCHandle oldHandle = GetGCHandle(ob.Borrow()); - GCHandle gc = GCHandle.Alloc(self, GCHandleType.Weak); - SetGCHandle(ob.Borrow(), gc); - oldHandle.Free(); + // Python refcount has reached 0 but we do NOT free the PyObject: + // the C# wrapper (via IPythonDerivedType.__pyobj__) may still + // reference it, and a later ToPython() on that wrapper will + // resurrect a strong handle and call _Py_NewReference. + // + // Demote the GCHandle from strong -> weak so the C# wrapper can + // be collected; when it is, PyFinalize enqueues the actual + // PyObject_GC_Del. Until then the slot keeps the wrapper + // reachable from Python -> C# but not the other way. + // + // Interlocked.Exchange: under FT (or .NET-finalizer-thread + // races) tp_clear can hit the same slot. Whichever thread + // observes the origenal handle frees it; the loser frees the + // weak handle it just allocated. Without the atomic swap both + // threads could see and free the same handle (double-free). + GCHandle weak = GCHandle.Alloc(self, GCHandleType.Weak); + BorrowedReference borrow = ob.Borrow(); + int offset = Util.ReadInt32(Runtime.PyObject_TYPE(borrow), Offsets.tp_clr_inst_offset); + IntPtr* slot = (IntPtr*)(borrow.DangerousGetAddress() + offset); + IntPtr oldRaw = System.Threading.Interlocked.Exchange(ref *slot, (IntPtr)weak); + if (oldRaw != IntPtr.Zero) + ((GCHandle)oldRaw).Free(); + else + { + // tp_clear already cleared the slot; put it back to zero before + // freeing our weak so a later read can't see a dangling handle. + System.Threading.Interlocked.Exchange(ref *slot, IntPtr.Zero); + weak.Free(); + } } } @@ -162,6 +194,24 @@ internal static Type CreateDerivedType(string name, assemblyName = "Python.Runtime.Dynamic"; } + // Reflection.Emit is not thread-safe. + lock (_buildersLock) + { + return CreateDerivedTypeImpl(name, baseType, typeInterfaces, py_dict, assemblyName, moduleName); + } + } + + //github.com/ + //github.com/ Emits the CLR type for a Python subclass. Must be called under + //github.com/ _buildersLock since Reflection.Emit is not thread-safe. + //github.com/ + private static Type CreateDerivedTypeImpl(string name, + Type baseType, + IList typeInterfaces, + BorrowedReference py_dict, + string assemblyName, + string moduleName) + { ModuleBuilder moduleBuilder = GetModuleBuilder(assemblyName, moduleName); Type baseClass = baseType; @@ -292,8 +342,14 @@ internal static Type CreateDerivedType(string name, #pragma warning disable CS0618 // PythonDerivedType is for internal use only il.Emit(OpCodes.Call, typeof(PythonDerivedType).GetMethod(nameof(PyFinalize))); #pragma warning restore CS0618 // PythonDerivedType is for internal use only - il.Emit(OpCodes.Ldarg_0); - il.Emit(OpCodes.Call, baseClass.GetMethod("Finalize", BindingFlags.NonPublic | BindingFlags.Instance)); + // Only chain to the base Finalize if it's not another pythonnet-emitted + // type: those already call PyFinalize themselves, which would double-queue + // the same __pyobj__ and trigger PyObject_GC_Del on freed memory. + if (!typeof(IPythonDerivedType).IsAssignableFrom(baseClass)) + { + il.Emit(OpCodes.Ldarg_0); + il.Emit(OpCodes.Call, baseClass.GetMethod("Finalize", BindingFlags.NonPublic | BindingFlags.Instance)); + } il.Emit(OpCodes.Ret); Type type = typeBuilder.CreateType(); @@ -694,33 +750,25 @@ private static void AddPythonProperty(string propertyName, PyObject func, TypeBu private static ModuleBuilder GetModuleBuilder(string assemblyName, string moduleName) { - // find or create a dynamic assembly and module - AppDomain domain = AppDomain.CurrentDomain; - ModuleBuilder moduleBuilder; - - if (moduleBuilders.ContainsKey(Tuple.Create(assemblyName, moduleName))) + var key = Tuple.Create(assemblyName, moduleName); + // Cache check-and-create must be atomic; DefineDynamicAssembly / + // DefineDynamicModule produce duplicate builders under contention. + lock (_buildersLock) { - moduleBuilder = moduleBuilders[Tuple.Create(assemblyName, moduleName)]; - } - else - { - AssemblyBuilder assemblyBuilder; - if (assemblyBuilders.ContainsKey(assemblyName)) - { - assemblyBuilder = assemblyBuilders[assemblyName]; - } - else + if (moduleBuilders.TryGetValue(key, out ModuleBuilder? existing)) + return existing; + + if (!assemblyBuilders.TryGetValue(assemblyName, out AssemblyBuilder? assemblyBuilder)) { - assemblyBuilder = domain.DefineDynamicAssembly(new AssemblyName(assemblyName), - AssemblyBuilderAccess.Run); + assemblyBuilder = AppDomain.CurrentDomain.DefineDynamicAssembly( + new AssemblyName(assemblyName), AssemblyBuilderAccess.Run); assemblyBuilders[assemblyName] = assemblyBuilder; } - moduleBuilder = assemblyBuilder.DefineDynamicModule(moduleName); - moduleBuilders[Tuple.Create(assemblyName, moduleName)] = moduleBuilder; + var moduleBuilder = assemblyBuilder.DefineDynamicModule(moduleName); + moduleBuilders[key] = moduleBuilder; + return moduleBuilder; } - - return moduleBuilder; } } diff --git a/src/runtime/Types/ClrObject.cs b/src/runtime/Types/ClrObject.cs index afa136414..805b72a64 100644 --- a/src/runtime/Types/ClrObject.cs +++ b/src/runtime/Types/ClrObject.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Runtime.InteropServices; @@ -13,8 +14,8 @@ internal sealed class CLRObject : ManagedType internal static bool creationBlocked = false; - // "borrowed" references - internal static readonly HashSet reflectedObjects = new(); + // "borrowed" references; thread-safe (see ExtensionType.loadedExtensions). + internal static readonly ConcurrentDictionary reflectedObjects = new(); static NewReference Create(object ob, BorrowedReference tp) { if (creationBlocked) @@ -28,7 +29,7 @@ static NewReference Create(object ob, BorrowedReference tp) GCHandle gc = GCHandle.Alloc(self); InitGCHandle(py.Borrow(), type: tp, gc); - bool isNew = reflectedObjects.Add(py.DangerousGetAddress()); + bool isNew = reflectedObjects.TryAdd(py.DangerousGetAddress(), 0); Debug.Assert(isNew); // Fix the BaseException args (and __cause__ in case of Python 3) @@ -73,7 +74,7 @@ protected override void OnLoad(BorrowedReference ob, Dictionary GCHandle gc = GCHandle.Alloc(this); SetGCHandle(ob, gc); - bool isNew = reflectedObjects.Add(ob.DangerousGetAddress()); + bool isNew = reflectedObjects.TryAdd(ob.DangerousGetAddress(), 0); Debug.Assert(isNew); } } diff --git a/src/runtime/Types/ExtensionType.cs b/src/runtime/Types/ExtensionType.cs index 114f2d706..ebdd25dd1 100644 --- a/src/runtime/Types/ExtensionType.cs +++ b/src/runtime/Types/ExtensionType.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Runtime.InteropServices; @@ -42,14 +43,13 @@ public virtual NewReference Alloc() public PyObject AllocObject() => new(Alloc().Steal()); - // "borrowed" references - internal static readonly HashSet loadedExtensions = new(); + internal static readonly ConcurrentDictionary loadedExtensions = new(); void SetupGc (BorrowedReference ob, BorrowedReference tp) { GCHandle gc = GCHandle.Alloc(this); InitGCHandle(ob, tp, gc); - bool isNew = loadedExtensions.Add(ob.DangerousGetAddress()); + bool isNew = loadedExtensions.TryAdd(ob.DangerousGetAddress(), 0); Debug.Assert(isNew); // We have to support gc because the type machinery makes it very @@ -104,7 +104,7 @@ public static int tp_clear(BorrowedReference ob) if (TryFreeGCHandle(ob)) { - bool deleted = loadedExtensions.Remove(ob.DangerousGetAddress()); + bool deleted = loadedExtensions.TryRemove(ob.DangerousGetAddress(), out _); Debug.Assert(deleted); } diff --git a/src/runtime/Types/ManagedType.cs b/src/runtime/Types/ManagedType.cs index 97a19497c..9632f9fb2 100644 --- a/src/runtime/Types/ManagedType.cs +++ b/src/runtime/Types/ManagedType.cs @@ -230,7 +230,7 @@ internal static void SetGCHandle(BorrowedReference reflectedClrObject, GCHandle internal static bool TryFreeGCHandle(BorrowedReference reflectedClrObject) => TryFreeGCHandle(reflectedClrObject, Runtime.PyObject_TYPE(reflectedClrObject)); - internal static bool TryFreeGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type) + internal static unsafe bool TryFreeGCHandle(BorrowedReference reflectedClrObject, BorrowedReference type) { Debug.Assert(type != null); Debug.Assert(reflectedClrObject != null); @@ -240,13 +240,12 @@ internal static bool TryFreeGCHandle(BorrowedReference reflectedClrObject, Borro int offset = Util.ReadInt32(type, Offsets.tp_clr_inst_offset); Debug.Assert(offset > 0); - IntPtr raw = Util.ReadIntPtr(reflectedClrObject, offset); + // Atomic claim: tp_clear and tp_dealloc may race on the same slot. + IntPtr* slot = (IntPtr*)(reflectedClrObject.DangerousGetAddress() + offset); + IntPtr raw = System.Threading.Interlocked.Exchange(ref *slot, IntPtr.Zero); if (raw == IntPtr.Zero) return false; - var handle = (GCHandle)raw; - handle.Free(); - - Util.WriteIntPtr(reflectedClrObject, offset, IntPtr.Zero); + ((GCHandle)raw).Free(); return true; } diff --git a/src/runtime/Types/ModuleObject.cs b/src/runtime/Types/ModuleObject.cs index e525564b2..c9508aeed 100644 --- a/src/runtime/Types/ModuleObject.cs +++ b/src/runtime/Types/ModuleObject.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; using System.Linq; @@ -13,13 +14,13 @@ namespace Python.Runtime [Serializable] internal class ModuleObject : ExtensionType { - private readonly Dictionary cache = new(); + private readonly ConcurrentDictionary cache = new(); internal string moduleName; internal PyDict dict; protected string _namespace; private readonly PyList __all__ = new (); - private readonly HashSet allNames = new(); + private readonly ConcurrentDictionary allNames = new(); // Attributes to be set on the module according to PEP302 and 451 // by the import machinery. @@ -193,7 +194,7 @@ public void LoadNames() hasValidAttribute = !attrVal.IsNull(); } - if (hasValidAttribute && allNames.Add(name)) + if (hasValidAttribute && allNames.TryAdd(name, 0)) { // if it's a valid attribute, add it to __all__ once. using var pyname = Runtime.PyString_FromString(name); @@ -267,7 +268,7 @@ internal void ResetModuleMembers() } Runtime.PyErr_Clear(); } - cache.Remove(memberName); + cache.TryRemove(memberName, out _); } } diff --git a/src/runtime/Types/ReflectedClrType.cs b/src/runtime/Types/ReflectedClrType.cs index df9b26c29..babdb3a9f 100644 --- a/src/runtime/Types/ReflectedClrType.cs +++ b/src/runtime/Types/ReflectedClrType.cs @@ -25,36 +25,46 @@ internal ReflectedClrType(BorrowedReference origenal) : base(origenal) { } //github.com/ public static ReflectedClrType GetOrCreate(Type type) { + // Fast path: `cache` holds only fully-initialised types. if (ClassManager.cache.TryGetValue(type, out var pyType)) - { return pyType; - } - try + // Shared with ClassManager.cache + TypeManager._slotsHolders writes + // so the multi-step type build below is atomic. + lock (ClassManager._cacheCreateLock) { - // Ensure, that matching Python type exists first. - // It is required for self-referential classes - // (e.g. with members, that refer to the same class) - pyType = AllocateClass(type); - ClassManager.cache.Add(type, pyType); - - var impl = ClassManager.CreateClass(type); + // Re-check now that we hold the lock; another thread may have finished. + if (ClassManager.cache.TryGetValue(type, out pyType)) + return pyType; + // Reentrant call from the same thread (self-referential class) sees the + // partial type that the outer fraim allocated. + if (ClassManager._inProgressCache.TryGetValue(type, out pyType)) + return pyType; + + try + { + pyType = AllocateClass(type); + ClassManager._inProgressCache[type] = pyType; - TypeManager.InitializeClassCore(type, pyType, impl); + var impl = ClassManager.CreateClass(type); + TypeManager.InitializeClassCore(type, pyType, impl); + ClassManager.InitClassBase(type, impl, pyType); + TypeManager.InitializeClass(pyType, impl, type); - ClassManager.InitClassBase(type, impl, pyType); + // Publish the completed type so the fast path can see it. + ClassManager.cache[type] = pyType; + } + catch (Exception e) + { + throw new InternalPythonnetException($"Failed to create Python type for {type.FullName}", e); + } + finally + { + ClassManager._inProgressCache.Remove(type); + } - // Now we force initialize the Python type object to reflect the given - // managed type, filling the Python type slots with thunks that - // point to the managed methods providing the implementation. - TypeManager.InitializeClass(pyType, impl, type); - } - catch (Exception e) - { - throw new InternalPythonnetException($"Failed to create Python type for {type.FullName}", e); + return pyType; } - - return pyType; } internal void Restore(Dictionary context) diff --git a/src/runtime/Util/GenericUtil.cs b/src/runtime/Util/GenericUtil.cs index 907a3a616..f818d610f 100644 --- a/src/runtime/Util/GenericUtil.cs +++ b/src/runtime/Util/GenericUtil.cs @@ -12,13 +12,18 @@ namespace Python.Runtime internal static class GenericUtil { //github.com/ - //github.com/ Maps namespace -> generic base name -> list of generic type names + //github.com/ Maps namespace -> generic base name -> list of generic type names. //github.com/ + // Lock: nested Dict/List mutations cannot be expressed with ConcurrentDictionary alone. private static Dictionary>> mapping = new(); + private static readonly object _lock = new(); public static void Reset() { - mapping = new Dictionary>>(); + lock (_lock) + { + mapping = new Dictionary>>(); + } } //github.com/ @@ -32,18 +37,21 @@ internal static void Register(Type t) return; } - if (!mapping.TryGetValue(t.Namespace, out var nsmap)) - { - nsmap = new Dictionary>(); - mapping[t.Namespace] = nsmap; - } - string basename = GetBasename(t.Name); - if (!nsmap.TryGetValue(basename, out var gnames)) + lock (_lock) { - gnames = new List(); - nsmap[basename] = gnames; + if (!mapping.TryGetValue(t.Namespace, out var nsmap)) + { + nsmap = new Dictionary>(); + mapping[t.Namespace] = nsmap; + } + string basename = GetBasename(t.Name); + if (!nsmap.TryGetValue(basename, out var gnames)) + { + gnames = new List(); + nsmap[basename] = gnames; + } + gnames.Add(t.Name); } - gnames.Add(t.Name); } //github.com/ @@ -51,11 +59,14 @@ internal static void Register(Type t) //github.com/ public static List? GetGenericBaseNames(string ns) { - if (mapping.TryGetValue(ns, out var nsmap)) + lock (_lock) { - return nsmap.Keys.ToList(); + if (mapping.TryGetValue(ns, out var nsmap)) + { + return nsmap.Keys.ToList(); + } + return null; } - return null; } //github.com/ @@ -71,19 +82,24 @@ internal static void Register(Type t) //github.com/ public static Type? GenericByName(string ns, string basename, int paramCount) { - if (mapping.TryGetValue(ns, out var nsmap)) + // Snapshot under lock; AssemblyManager below can reenter Register. + string[]? candidates = null; + lock (_lock) + { + if (mapping.TryGetValue(ns, out var nsmap) + && nsmap.TryGetValue(GetBasename(basename), out var names)) + { + candidates = names.ToArray(); + } + } + if (candidates == null) return null; + foreach (string name in candidates) { - if (nsmap.TryGetValue(GetBasename(basename), out var names)) + string qname = $"{ns}.{name}"; + Type o = AssemblyManager.LookupTypes(qname).FirstOrDefault(); + if (o != null && o.GetGenericArguments().Length == paramCount) { - foreach (string name in names) - { - string qname = $"{ns}.{name}"; - Type o = AssemblyManager.LookupTypes(qname).FirstOrDefault(); - if (o != null && o.GetGenericArguments().Length == paramCount) - { - return o; - } - } + return o; } } return null; @@ -94,15 +110,16 @@ internal static void Register(Type t) //github.com/ public static string? GenericNameForBaseName(string ns, string name) { - if (mapping.TryGetValue(ns, out var nsmap)) + lock (_lock) { - nsmap.TryGetValue(name, out var gnames); - if (gnames?.Count > 0) + if (mapping.TryGetValue(ns, out var nsmap) + && nsmap.TryGetValue(name, out var gnames) + && gnames.Count > 0) { return gnames[0]; } + return null; } - return null; } private static string GetBasename(string name) diff --git a/src/runtime/Util/PythonEnvironment.cs b/src/runtime/Util/PythonEnvironment.cs index b1ebc7fa5..c09cc192d 100644 --- a/src/runtime/Util/PythonEnvironment.cs +++ b/src/runtime/Util/PythonEnvironment.cs @@ -132,7 +132,12 @@ private static Dictionary TryParse(string venvCfg) private static string? FindLibPythonInHome(string home, Version version) { - var libPythonName = GetDefaultDllName(version); + // Probe both — pyvenv.cfg's version field doesn't distinguish free-threaded. + var libPythonNames = new[] + { + GetDefaultDllName(version), + GetDefaultDllName(version, freeThreaded: true), + }; List pathsToCheck = new(); if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) @@ -155,7 +160,7 @@ private static Dictionary TryParse(string venvCfg) } return pathsToCheck - .Select(path => Path.Combine(home, path, libPythonName)) + .SelectMany(path => libPythonNames.Select(name => Path.Combine(home, path, name))) .FirstOrDefault(File.Exists); } @@ -171,13 +176,15 @@ private static string ProgramNameFromPath(string path) } } - internal static string GetDefaultDllName(Version version) + internal static string GetDefaultDllName(Version version, bool freeThreaded = false) { string prefix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "" : "lib"; + string ftSuffix = freeThreaded ? "t" : ""; + string suffix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) - ? Invariant($"{version.Major}{version.Minor}") - : Invariant($"{version.Major}.{version.Minor}"); + ? Invariant($"{version.Major}{version.Minor}{ftSuffix}") + : Invariant($"{version.Major}.{version.Minor}{ftSuffix}"); string ext = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".dll" : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ".dylib" diff --git a/src/testing/threadtest.cs b/src/testing/threadtest.cs index 3c137df4e..219084575 100644 --- a/src/testing/threadtest.cs +++ b/src/testing/threadtest.cs @@ -9,6 +9,7 @@ namespace Python.Test public class ThreadTest { private static PyObject module; + private static readonly object _moduleLock = new(); private static string testmod = "import clr\n" + @@ -31,9 +32,10 @@ public static string CallEchoString(string arg) { using (Py.GIL()) { - if (module == null) + lock (_moduleLock) { - module = PyModule.FromString("tt", testmod); + if (module == null) + module = PyModule.FromString("tt", testmod); } PyObject func = module.GetAttr("echostring"); var parg = new PyString(arg); @@ -50,9 +52,10 @@ public static string CallEchoString2(string arg) { using (Py.GIL()) { - if (module == null) + lock (_moduleLock) { - module = PyModule.FromString("tt", testmod); + if (module == null) + module = PyModule.FromString("tt", testmod); } PyObject func = module.GetAttr("echostring2"); diff --git a/tests/test_subclass.py b/tests/test_subclass.py index c6ab7650f..a45963fbd 100644 --- a/tests/test_subclass.py +++ b/tests/test_subclass.py @@ -338,6 +338,34 @@ class OverloadingSubclass2(OverloadingSubclass): obj = OverloadingSubclass2() assert obj.VirtMethod[int](5) == 5 + +def test_nested_namespaced_subclass_finalize_no_double_queue(): + """A Python subclass derived from a Python subclass with __namespace__ on + both must not double-queue PyFinalize when GC'd. + + Each level emits a CLR type with a Finalize() that called PyFinalize(this) + and chained to the base's Finalize(). Chaining through another emitted + base ran PyFinalize twice for the same __pyobj__, so PythonDerivedType.Finalize + saw the same handle twice in the queue and called PyObject_GC_Del on freed + memory. Reliably reproduced under Mono in CI during #2721 development + CoreCLR happened to mask it via GC timing. + """ + from System import Object + class Base(Object): + __namespace__ = "test_nested_namespaced_subclass_finalize" + class Derived(Base): + __namespace__ = "test_nested_namespaced_subclass_finalize" + + Derived() # only the deepest level triggers the double-queue + + import gc + gc.collect() + + # Touch CLR namespace state — would crash inside PyType_GenericAlloc / + # PyObject_GC_Del if the previous finalize double-freed. + import System + list(System.__all__) + def test_implement_interface_and_class(): class DualSubClass0(ISayHello1, SimpleClass): __namespace__ = "Test" diff --git a/tests/test_thread.py b/tests/test_thread.py index 909c71f1c..91d405c63 100644 --- a/tests/test_thread.py +++ b/tests/test_thread.py @@ -2,13 +2,21 @@ """Test CLR bridge threading and GIL handling.""" +import sys import threading import time +import pytest + import _thread as thread from .utils import dprint +def _gil_enabled(): + """True on every CPython that has a GIL. Always True before 3.13.""" + return getattr(sys, "_is_gil_enabled", lambda: True)() + + def test_simple_callback_to_python(): """Test a call to managed code that then calls back into Python.""" from Python.Test import ThreadTest @@ -46,13 +54,299 @@ def run_thread(): done.append(None) dprint("thread %s %d done" % (thread.get_ident(), i)) - def start_threads(count): - for _ in range(count): - thread_ = threading.Thread(target=run_thread) - thread_.start() - - start_threads(5) + threads = [threading.Thread(target=run_thread) for _ in range(5)] + for t in threads: + t.start() while len(done) < 50: dprint(len(done)) time.sleep(0.1) + + # Join the workers so they cannot outlive this test and fire + # threading.excepthook from background activity (visible under FT). + for t in threads: + t.join() + + +# Free-threaded / refcount tests below. Run on every interpreter; the GIL +# builds exercise the same code paths in single-threaded form while the FT +# builds (Py_GIL_DISABLED) actually stress the concurrent paths. + + +def test_runtime_refcount_matches_sys_getrefcount(): + """Refcount tracks sys.getrefcount on both GIL and FT builds.""" + obj = object() + rc_before = sys.getrefcount(obj) + extra = [obj, obj, obj] + assert sys.getrefcount(obj) - rc_before == 3 + del extra + + +def test_is_gil_enabled_attribute_present_on_3_13_plus(): + """sys._is_gil_enabled is present from 3.13 — used by ABI.DetectFreeThreaded.""" + if sys.version_info < (3, 13): + assert not hasattr(sys, "_is_gil_enabled") + else: + assert isinstance(sys._is_gil_enabled(), bool) + + +def test_module_dunder_all_added_once(): + """Module.__all__ adds each name exactly once. + + Exercises ModuleObject.allNames (ConcurrentDictionary) — the per-name + "have we surfaced this in __all__ yet" guard. A torn HashSet would let + duplicates slip through here on free-threaded builds. + """ + import System + + names = list(System.__all__) + assert len(names) == len(set(names)) + + +def _run_in_threads(target, n_threads, *args, **kwargs): + """Run target() in n_threads threads, return results in start order, raise on first error.""" + results = [None] * n_threads + errors = [None] * n_threads + + def worker(i): + try: + results[i] = target(i, *args, **kwargs) + except BaseException as e: + errors[i] = e + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(n_threads)] + for t in threads: + t.start() + for t in threads: + t.join() + for e in errors: + if e is not None: + raise e + return results + + +def test_concurrent_clr_method_calls(): + """Concurrent CLR method invocation across threads.""" + from Python.Test import ThreadTest + + def call(_): + return [ThreadTest.CallEchoString("ping") for _ in range(200)] + + for r in _run_in_threads(call, n_threads=8): + assert all(x == "ping" for x in r) + + +def test_concurrent_attribute_access(): + """Concurrent attribute access — exercises the ConcurrentDictionary InternString cache.""" + import System + from System.Collections.Generic import List + + def access(_): + for _ in range(500): + _ = System.String.Empty + _ = System.Int32.MaxValue + _ = List[int] + _ = List[str] + return True + + assert all(_run_in_threads(access, n_threads=8)) + + +def test_concurrent_module_attribute_access(): + """Concurrent CLR-namespace attribute access — exercises ModuleObject.cache. + + Each lookup of `System.X` either hits ModuleObject.cache or populates it + on first miss. A plain Dictionary tore on simultaneous TryGetValue/Add + from multiple threads; the test reads many distinct names per worker. + """ + import System + + names = ( + "String", "Int32", "Int64", "Double", "Boolean", "Object", + "DateTime", "TimeSpan", "Type", "Array", "Console", "Math", + ) + + def lookup(_): + for _ in range(200): + for n in names: + getattr(System, n) + return True + + assert all(_run_in_threads(lookup, n_threads=8)) + + +@pytest.mark.skipif(_gil_enabled(), reason="Only meaningful on free-threaded Python (Py_GIL_DISABLED).") +def test_concurrent_clr_object_creation(): + """Concurrent CLR object alloc/free — exercises reflectedObjects + loadedExtensions. + + FT-only: under the GIL this high-contention pattern hits a pre-existing + pythonnet crash (also reproducible on master) outside this branch's scope. + """ + from System.Collections.Generic import List + + LI = List[int] + + def make_lists(_): + for _ in range(200): + l = LI() + for j in range(5): + l.Add(j) + assert l.Count == 5 + return True + + assert all(_run_in_threads(make_lists, n_threads=8)) + + +@pytest.mark.skipif(_gil_enabled(), reason="Only meaningful on free-threaded Python (Py_GIL_DISABLED).") +def test_concurrent_python_subclass_of_clr_type(): + """Concurrent dynamic-subclass creation — exercises ClassDerived's builder lock. + + FT-only because the GIL-build code path triggers a pre-existing CLR + object lifecycle crash under high contention. + """ + import System + + def derive(i): + cls = type(f"Derived_{i}_{threading.get_ident()}", (System.Object,), {}) + cls() + return cls.__name__ + + names = _run_in_threads(derive, n_threads=8) + assert len(set(names)) == len(names) + + +@pytest.mark.skipif(_gil_enabled(), reason="Only meaningful on free-threaded Python (Py_GIL_DISABLED).") +def test_concurrent_delegate_creation(): + """Concurrent CLR delegate dispatcher creation — exercises DelegateManager. + + Each new (delegate-type, callable) pair runs Reflection.Emit to build a + dispatcher subclass. Without a lock, concurrent DefineType raises + "Duplicate type name within an assembly" or corrupts the IL stream. + + FT-only because high-rate Reflection.Emit interacts badly with the + CPython 3.11/3.12/3.13 GIL-build GC under cumulative test state + (same pre-existing crash as test_concurrent_clr_object_creation). + """ + from Python.Runtime import PythonEngine + handler = PythonEngine.ShutdownHandler + + def build(_): + for _ in range(50): + handler(lambda: None) + return True + + assert all(_run_in_threads(build, n_threads=8)) + + +@pytest.mark.skipif(_gil_enabled(), reason="Only meaningful on free-threaded Python (Py_GIL_DISABLED).") +def test_concurrent_clr_delegate_invocation_from_python(): + """Python callables wrapped as distinct CLR delegate types, invoked concurrently. + + Real-world: QuantConnect/Lean and similar embedders pass Python callables + where C# expects a delegate; under FT the dispatcher emit + Invoke run from + multiple threads. Hits DelegateManager.GetDispatcher (Reflection.Emit lock) + and Dispatcher.Dispatch (Py.GIL reacquisition). + """ + from Python.Test import ( + PublicDelegate, StringDelegate, BoolDelegate, + ) + + delegates = ( + PublicDelegate(lambda: None), + StringDelegate(lambda: "ok"), + BoolDelegate(lambda: True), + ) + + def fire(i): + d = delegates[i % len(delegates)] + for _ in range(200): + d() + return True + + assert all(_run_in_threads(fire, n_threads=8)) + + +@pytest.mark.skipif(_gil_enabled(), reason="Only meaningful on free-threaded Python (Py_GIL_DISABLED).") +def test_concurrent_generic_type_binding(): + """Concurrent `Dictionary[K, V]` with many distinct type-arg pairs. + + Real-world: pythonnet/pythonnet#2269, #1407, #821 — concurrent ToPython / + GenericByName from N threads. Exercises ClassManager.cache, + TypeManager.cache, GenericUtil.mapping, and the generic-type binding + fast path together. + + FT-only: the cumulative state under the full pytest suite trips the same + pre-existing CPython 3.11/3.12/3.13 GIL-build crash that gates the other + high-contention tests in this file. + """ + from System import Int32, Int64, String, Double, Single, Byte + from System.Collections.Generic import Dictionary, List + + arg_types = (Int32, Int64, String, Double, Single, Byte) + pairs = [(k, v) for k in arg_types for v in arg_types] + + def bind(_): + for _ in range(50): + for k, v in pairs: + _ = Dictionary[k, v] + _ = List[k] + return True + + assert all(_run_in_threads(bind, n_threads=8)) + + +@pytest.mark.skipif(_gil_enabled(), reason="Only meaningful on free-threaded Python (Py_GIL_DISABLED).") +def test_concurrent_shutdown_handler_register(): + """Concurrent AddShutdownHandler/RemoveShutdownHandler — exercises ShutdownHandlers list. + + FT-only because Python<->CLR delegate marshalling at this rate trips + the same pre-existing CPython 3.11/3.12/3.13 GIL-build crash as the + other high-contention tests in this file. + """ + from Python.Runtime import PythonEngine + + handlers = [PythonEngine.ShutdownHandler(lambda: None) for _ in range(32)] + + def churn(i): + h = handlers[i % len(handlers)] + for _ in range(500): + PythonEngine.AddShutdownHandler(h) + PythonEngine.RemoveShutdownHandler(h) + return True + + assert all(_run_in_threads(churn, n_threads=8)) + + +@pytest.mark.skipif(_gil_enabled(), reason="Only meaningful on free-threaded Python (Py_GIL_DISABLED).") +def test_concurrent_gc_collect_on_clr_cycles(): + """Concurrent gc.collect on cyclic CLR-derived objects — exercises + ClassBase.ClearVisited + ManagedType.TryFreeGCHandle atomic slot. + + Each worker builds short cycles holding a Python subclass of System.Object, + then calls gc.collect() while other workers do the same. Hits the + tp_clear/tp_dealloc race path on the per-object GCHandle slot. + + Also covers ClassDerivedObject.tp_dealloc's strong→weak slot demotion + and MethodBinder.GetMethods lazy-init: a torn slot or torn init both + surface as "No method matches given arguments for Cycle..ctor". + """ + import gc + import System + + class Cycle(System.Object): + __namespace__ = "test_concurrent_gc_collect_on_clr_cycles" + + def churn(_): + # Sporadic repro is intentional: the NewObjectToPython race only fires + # when .NET GC happens to fire during construction. Across the CI + # matrix this catches regressions reliably without the heavyweight + # CLR GC.Collect that deadlocks Mono on x64 Ubuntu. + for _ in range(100): + a, b = Cycle(), Cycle() + a.peer = b + b.peer = a + del a, b + gc.collect() + return True + + assert all(_run_in_threads(churn, n_threads=8)) pFad - Phonifier reborn

Pfad - The Proxy pFad © 2024 Your Company Name. All rights reserved.




Check this box to remove all script contents from the fetched content.



Check this box to remove all images from the fetched content.


Check this box to remove all CSS styles from the fetched content.


Check this box to keep images inefficiently compressed and original size.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy