Isolate worker panics and preserve queued tasks for surviving workers

This commit is contained in:
2026-10-20 21:47:55 +08:00
parent 5bf542fc67
commit 44cadc6d26
3 changed files with 286 additions and 37 deletions

84
pool.go
View File

@@ -85,9 +85,9 @@ func (p *Pool) Submit(task func()) error {
}
// Close stops accepting tasks and waits for workers to exit. Tasks accepted
// before Close normally finish before it returns. If a task calls
// runtime.Goexit and that causes the last worker to exit, queued tasks are
// discarded. It is safe to call Close concurrently or more than once.
// before Close normally finish before it returns. If runtime.Goexit or a
// worker panic exits the last worker, queued tasks are discarded. It is safe
// to call Close concurrently or more than once.
func (p *Pool) Close() {
p.mu.Lock()
p.closed = true
@@ -120,6 +120,9 @@ func (p *Pool) startWorkerLocked(task func(), withTask bool) {
func (p *Pool) run(w *worker) {
normalExit := false
defer func() {
if recovered := recover(); recovered != nil {
safeLogf("worker panic: %v", recovered)
}
if !normalExit {
p.workerExitedAbnormally()
}
@@ -127,41 +130,55 @@ func (p *Pool) run(w *worker) {
for {
task, ok := <-w.inbox
if !ok {
p.mu.Lock()
p.total--
p.workerExitedLocked()
normalExit = true
p.mu.Unlock()
p.finishClosedWorker(&normalExit)
return
}
runTask(task)
p.mu.Lock()
if p.queueLen > 0 {
next := p.dequeueLocked()
w.inbox <- next
p.submitCond.Signal()
p.mu.Unlock()
continue
}
otherIdle := len(p.idle)
if p.closed || (p.total > p.min && otherIdle > p.min/2) {
p.total--
normalExit = true
if !p.closed {
p.submitCond.Signal()
} else {
p.workerExitedLocked()
}
p.mu.Unlock()
if p.finishTask(w, &normalExit) {
return
}
p.idle = append(p.idle, w)
p.submitCond.Signal()
p.mu.Unlock()
}
}
// finishClosedWorker accounts for a normal Close of an idle worker.
func (p *Pool) finishClosedWorker(normalExit *bool) {
p.mu.Lock()
defer p.mu.Unlock()
p.total--
*normalExit = true
p.workerExitedLocked()
}
// finishTask commits the post-task transition under mu. If queue work exists,
// it first hands off the head item and only then removes it from the ring, so
// a closed inbox panic cannot lose an accepted task. The deferred unlock runs
// before run's panic cleanup acquires the mutex.
func (p *Pool) finishTask(w *worker, normalExit *bool) bool {
p.mu.Lock()
defer p.mu.Unlock()
if p.queueLen > 0 {
next := p.queue[p.queueHead]
w.inbox <- next
p.dequeueLocked()
p.submitCond.Signal()
return false
}
otherIdle := len(p.idle)
if p.closed || (p.total > p.min && otherIdle > p.min/2) {
p.total--
*normalExit = true
if !p.closed {
p.submitCond.Signal()
} else {
p.workerExitedLocked()
}
return true
}
p.idle = append(p.idle, w)
p.submitCond.Signal()
return false
}
// enqueueLocked and dequeueLocked are called with mu held. The queue is a
// fixed-capacity FIFO ring; clearing the dequeued slot releases its closure.
func (p *Pool) enqueueLocked(task func()) {
@@ -212,10 +229,15 @@ func (p *Pool) workerExitedAbnormally() {
p.mu.Unlock()
}
func safeLogf(format string, args ...any) {
defer func() { _ = recover() }()
log.Printf(format, args...)
}
func runTask(task func()) {
defer func() {
if recovered := recover(); recovered != nil {
log.Printf("worker task panic: %v", recovered)
safeLogf("worker task panic: %v", recovered)
}
}()
task()