gh-115634: Fix ProcessPoolExecutor deadlock with max_tasks_per_child … · python/cpython@b706767

GitHub

@@ -360,7 +360,7 @@ def run(self):

360360ifexecutor:=self.executor_reference():

361361ifprocess_exited:

362362withself.shutdown_lock:

363-executor._adjust_process_count()

363+executor._replace_dead_worker()

364364else:

365365executor._idle_worker_semaphore.release()

366366delexecutor

@@ -772,6 +772,30 @@ def _start_executor_manager_thread(self):

772772_threads_wakeups[self._executor_manager_thread] = \

773773self._executor_manager_thread_wakeup

774774775+def_replace_dead_worker(self):

776+# gh-132969: avoid error when state is reset and executor is still running,

777+# which will happen when shutdown(wait=False) is called.

778+ifself._processesisNone:

779+return

780+781+# A replacement is pointless when shutting down with nothing left

782+# to run. Both attributes are read under _shutdown_lock, which

783+# shutdown() holds while setting _shutdown_thread.

784+assertself._shutdown_lock.locked()

785+ifself._shutdown_threadandnotself._pending_work_items:

786+return

787+788+# gh-115634: A worker exited after reaching max_tasks_per_child and

789+# has been removed from self._processes. Do not consult

790+# _idle_worker_semaphore here: it counts task completions, not idle

791+# workers, so it can hold a stale token released by the now-dead

792+# worker. Trusting such a token would leave the pool a worker short,

793+# deadlocking once all workers reach their task limit. Spawning is

794+# safe from this (manager) thread despite gh-90622 because

795+# max_tasks_per_child is rejected for the "fork" start method.

796+iflen(self._processes) <self._max_workers:

797+self._spawn_process()

798+775799def_adjust_process_count(self):

776800# gh-132969: avoid error when state is reset and executor is still running,

777801# which will happen when shutdown(wait=False) is called.

@@ -784,12 +808,12 @@ def _adjust_process_count(self):

784808785809process_count=len(self._processes)

786810ifprocess_count<self._max_workers:

787-# Assertion disabled as this codepath is also used to replace a

788-# worker that unexpectedly dies, even when using the 'fork' start

789-# method. That means there is still a potential deadlock bug. If a

790-# 'fork' mp_context worker dies, we'll be forking a new one when

791-# we know a thread is running (self._executor_manager_thread).

792-#assert self._safe_to_dynamically_spawn_children or not self._executor_manager_thread, 'https://github.com/python/cpython/issues/90622'

811+# gh-90622: spawning a child via fork while another thread is

812+# running can deadlock in the child. submit() only calls this

813+# method when using a non-fork start method.

814+assert (self._safe_to_dynamically_spawn_children

815+ornotself._executor_manager_thread), (

816+'https://github.com/python/cpython/issues/90622')

793817self._spawn_process()

794818795819def_launch_processes(self):