[3.14] gh-115634: Fix ProcessPoolExecutor deadlock with max_tasks_per… · python/cpython@42f81fa

GitHub

@@ -360,7 +360,7 @@ def run(self):

360360ifexecutor:=self.executor_reference():

361361ifprocess_exited:

362362withself.shutdown_lock:

363-executor._adjust_process_count()

363+executor._replace_dead_worker()

364364else:

365365executor._idle_worker_semaphore.release()

366366delexecutor

@@ -759,6 +759,30 @@ def _start_executor_manager_thread(self):

759759_threads_wakeups[self._executor_manager_thread] = \

760760self._executor_manager_thread_wakeup

761761762+def_replace_dead_worker(self):

763+# gh-132969: avoid error when state is reset and executor is still running,

764+# which will happen when shutdown(wait=False) is called.

765+ifself._processesisNone:

766+return

767+768+# A replacement is pointless when shutting down with nothing left

769+# to run. Both attributes are read under _shutdown_lock, which

770+# shutdown() holds while setting _shutdown_thread.

771+assertself._shutdown_lock.locked()

772+ifself._shutdown_threadandnotself._pending_work_items:

773+return

774+775+# gh-115634: A worker exited after reaching max_tasks_per_child and

776+# has been removed from self._processes. Do not consult

777+# _idle_worker_semaphore here: it counts task completions, not idle

778+# workers, so it can hold a stale token released by the now-dead

779+# worker. Trusting such a token would leave the pool a worker short,

780+# deadlocking once all workers reach their task limit. Spawning is

781+# safe from this (manager) thread despite gh-90622 because

782+# max_tasks_per_child is rejected for the "fork" start method.

783+iflen(self._processes) <self._max_workers:

784+self._spawn_process()

785+762786def_adjust_process_count(self):

763787# gh-132969: avoid error when state is reset and executor is still running,

764788# which will happen when shutdown(wait=False) is called.

@@ -771,12 +795,12 @@ def _adjust_process_count(self):

771795772796process_count=len(self._processes)

773797ifprocess_count<self._max_workers:

774-# Assertion disabled as this codepath is also used to replace a

775-# worker that unexpectedly dies, even when using the 'fork' start

776-# method. That means there is still a potential deadlock bug. If a

777-# 'fork' mp_context worker dies, we'll be forking a new one when

778-# we know a thread is running (self._executor_manager_thread).

779-#assert self._safe_to_dynamically_spawn_children or not self._executor_manager_thread, 'https://github.com/python/cpython/issues/90622'

798+# gh-90622: spawning a child via fork while another thread is

799+# running can deadlock in the child. submit() only calls this

800+# method when using a non-fork start method.

801+assert (self._safe_to_dynamically_spawn_children

802+ornotself._executor_manager_thread), (

803+'https://github.com/python/cpython/issues/90622')

780804self._spawn_process()

781805782806def_launch_processes(self):