Isolate worker panics and preserve queued tasks for surviving workers
This commit is contained in:
84
pool.go
84
pool.go
@@ -85,9 +85,9 @@ func (p *Pool) Submit(task func()) error {
|
||||
}
|
||||
|
||||
// Close stops accepting tasks and waits for workers to exit. Tasks accepted
|
||||
// before Close normally finish before it returns. If a task calls
|
||||
// runtime.Goexit and that causes the last worker to exit, queued tasks are
|
||||
// discarded. It is safe to call Close concurrently or more than once.
|
||||
// before Close normally finish before it returns. If runtime.Goexit or a
|
||||
// worker panic exits the last worker, queued tasks are discarded. It is safe
|
||||
// to call Close concurrently or more than once.
|
||||
func (p *Pool) Close() {
|
||||
p.mu.Lock()
|
||||
p.closed = true
|
||||
@@ -120,6 +120,9 @@ func (p *Pool) startWorkerLocked(task func(), withTask bool) {
|
||||
func (p *Pool) run(w *worker) {
|
||||
normalExit := false
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
safeLogf("worker panic: %v", recovered)
|
||||
}
|
||||
if !normalExit {
|
||||
p.workerExitedAbnormally()
|
||||
}
|
||||
@@ -127,41 +130,55 @@ func (p *Pool) run(w *worker) {
|
||||
for {
|
||||
task, ok := <-w.inbox
|
||||
if !ok {
|
||||
p.mu.Lock()
|
||||
p.total--
|
||||
p.workerExitedLocked()
|
||||
normalExit = true
|
||||
p.mu.Unlock()
|
||||
p.finishClosedWorker(&normalExit)
|
||||
return
|
||||
}
|
||||
runTask(task)
|
||||
|
||||
p.mu.Lock()
|
||||
if p.queueLen > 0 {
|
||||
next := p.dequeueLocked()
|
||||
w.inbox <- next
|
||||
p.submitCond.Signal()
|
||||
p.mu.Unlock()
|
||||
continue
|
||||
}
|
||||
otherIdle := len(p.idle)
|
||||
if p.closed || (p.total > p.min && otherIdle > p.min/2) {
|
||||
p.total--
|
||||
normalExit = true
|
||||
if !p.closed {
|
||||
p.submitCond.Signal()
|
||||
} else {
|
||||
p.workerExitedLocked()
|
||||
}
|
||||
p.mu.Unlock()
|
||||
if p.finishTask(w, &normalExit) {
|
||||
return
|
||||
}
|
||||
p.idle = append(p.idle, w)
|
||||
p.submitCond.Signal()
|
||||
p.mu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// finishClosedWorker accounts for a normal Close of an idle worker.
|
||||
func (p *Pool) finishClosedWorker(normalExit *bool) {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
p.total--
|
||||
*normalExit = true
|
||||
p.workerExitedLocked()
|
||||
}
|
||||
|
||||
// finishTask commits the post-task transition under mu. If queue work exists,
|
||||
// it first hands off the head item and only then removes it from the ring, so
|
||||
// a closed inbox panic cannot lose an accepted task. The deferred unlock runs
|
||||
// before run's panic cleanup acquires the mutex.
|
||||
func (p *Pool) finishTask(w *worker, normalExit *bool) bool {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if p.queueLen > 0 {
|
||||
next := p.queue[p.queueHead]
|
||||
w.inbox <- next
|
||||
p.dequeueLocked()
|
||||
p.submitCond.Signal()
|
||||
return false
|
||||
}
|
||||
otherIdle := len(p.idle)
|
||||
if p.closed || (p.total > p.min && otherIdle > p.min/2) {
|
||||
p.total--
|
||||
*normalExit = true
|
||||
if !p.closed {
|
||||
p.submitCond.Signal()
|
||||
} else {
|
||||
p.workerExitedLocked()
|
||||
}
|
||||
return true
|
||||
}
|
||||
p.idle = append(p.idle, w)
|
||||
p.submitCond.Signal()
|
||||
return false
|
||||
}
|
||||
|
||||
// enqueueLocked and dequeueLocked are called with mu held. The queue is a
|
||||
// fixed-capacity FIFO ring; clearing the dequeued slot releases its closure.
|
||||
func (p *Pool) enqueueLocked(task func()) {
|
||||
@@ -212,10 +229,15 @@ func (p *Pool) workerExitedAbnormally() {
|
||||
p.mu.Unlock()
|
||||
}
|
||||
|
||||
func safeLogf(format string, args ...any) {
|
||||
defer func() { _ = recover() }()
|
||||
log.Printf(format, args...)
|
||||
}
|
||||
|
||||
func runTask(task func()) {
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
log.Printf("worker task panic: %v", recovered)
|
||||
safeLogf("worker task panic: %v", recovered)
|
||||
}
|
||||
}()
|
||||
task()
|
||||
|
||||
Reference in New Issue
Block a user