forked from forgejo/forgejo
Rewrite queue (#24505)
# ⚠️ Breaking Many deprecated queue config options are removed (actually, they should have been removed in 1.18/1.19). If you see the fatal message when starting Gitea: "Please update your app.ini to remove deprecated config options", please follow the error messages to remove these options from your app.ini. Example: ``` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options ``` Many options in `[queue]` are are dropped, including: `WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`, `BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed from app.ini. # The problem The old queue package has some legacy problems: * complexity: I doubt few people could tell how it works. * maintainability: Too many channels and mutex/cond are mixed together, too many different structs/interfaces depends each other. * stability: due to the complexity & maintainability, sometimes there are strange bugs and difficult to debug, and some code doesn't have test (indeed some code is difficult to test because a lot of things are mixed together). * general applicability: although it is called "queue", its behavior is not a well-known queue. * scalability: it doesn't seem easy to make it work with a cluster without breaking its behaviors. It came from some very old code to "avoid breaking", however, its technical debt is too heavy now. It's a good time to introduce a better "queue" package. # The new queue package It keeps using old config and concept as much as possible. * It only contains two major kinds of concepts: * The "base queue": channel, levelqueue, redis * They have the same abstraction, the same interface, and they are tested by the same testing code. * The "WokerPoolQueue", it uses the "base queue" to provide "worker pool" function, calls the "handler" to process the data in the base queue. * The new code doesn't do "PushBack" * Think about a queue with many workers, the "PushBack" can't guarantee the order for re-queued unhandled items, so in new code it just does "normal push" * The new code doesn't do "pause/resume" * The "pause/resume" was designed to handle some handler's failure: eg: document indexer (elasticsearch) is down * If a queue is paused for long time, either the producers blocks or the new items are dropped. * The new code doesn't do such "pause/resume" trick, it's not a common queue's behavior and it doesn't help much. * If there are unhandled items, the "push" function just blocks for a few seconds and then re-queue them and retry. * The new code doesn't do "worker booster" * Gitea's queue's handlers are light functions, the cost is only the go-routine, so it doesn't make sense to "boost" them. * The new code only use "max worker number" to limit the concurrent workers. * The new "Push" never blocks forever * Instead of creating more and more blocking goroutines, return an error is more friendly to the server and to the end user. There are more details in code comments: eg: the "Flush" problem, the strange "code.index" hanging problem, the "immediate" queue problem. Almost ready for review. TODO: * [x] add some necessary comments during review * [x] add some more tests if necessary * [x] update documents and config options * [x] test max worker / active worker * [x] re-run the CI tasks to see whether any test is flaky * [x] improve the `handleOldLengthConfiguration` to provide more friendly messages * [x] fine tune default config values (eg: length?) ## Code coverage: 
This commit is contained in:
wxiaoguang
GitHub
parent
cb700aedd1
commit
6f9c278559
100 changed files with 2496 additions and 6858 deletions
260
modules/queue/workerqueue_test.go
Normal file
260
modules/queue/workerqueue_test.go
Normal file
|
|
@ -0,0 +1,260 @@
|
|||
// Copyright 2023 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package queue
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func runWorkerPoolQueue[T any](q *WorkerPoolQueue[T]) func() {
|
||||
var stop func()
|
||||
started := make(chan struct{})
|
||||
stopped := make(chan struct{})
|
||||
go func() {
|
||||
q.Run(func(f func()) { stop = f; close(started) }, nil)
|
||||
close(stopped)
|
||||
}()
|
||||
<-started
|
||||
return func() {
|
||||
stop()
|
||||
<-stopped
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerPoolQueueUnhandled(t *testing.T) {
|
||||
oldUnhandledItemRequeueDuration := unhandledItemRequeueDuration.Load()
|
||||
unhandledItemRequeueDuration.Store(0)
|
||||
defer unhandledItemRequeueDuration.Store(oldUnhandledItemRequeueDuration)
|
||||
|
||||
mu := sync.Mutex{}
|
||||
|
||||
test := func(t *testing.T, queueSetting setting.QueueSettings) {
|
||||
queueSetting.Length = 100
|
||||
queueSetting.Type = "channel"
|
||||
queueSetting.Datadir = t.TempDir() + "/test-queue"
|
||||
m := map[int]int{}
|
||||
|
||||
// odds are handled once, evens are handled twice
|
||||
handler := func(items ...int) (unhandled []int) {
|
||||
testRecorder.Record("handle:%v", items)
|
||||
for _, item := range items {
|
||||
mu.Lock()
|
||||
if item%2 == 0 && m[item] == 0 {
|
||||
unhandled = append(unhandled, item)
|
||||
}
|
||||
m[item]++
|
||||
mu.Unlock()
|
||||
}
|
||||
return unhandled
|
||||
}
|
||||
|
||||
q, _ := NewWorkerPoolQueueBySetting("test-workpoolqueue", queueSetting, handler, false)
|
||||
stop := runWorkerPoolQueue(q)
|
||||
for i := 0; i < queueSetting.Length; i++ {
|
||||
testRecorder.Record("push:%v", i)
|
||||
assert.NoError(t, q.Push(i))
|
||||
}
|
||||
assert.NoError(t, q.FlushWithContext(context.Background(), 0))
|
||||
stop()
|
||||
|
||||
ok := true
|
||||
for i := 0; i < queueSetting.Length; i++ {
|
||||
if i%2 == 0 {
|
||||
ok = ok && assert.EqualValues(t, 2, m[i], "test %s: item %d", t.Name(), i)
|
||||
} else {
|
||||
ok = ok && assert.EqualValues(t, 1, m[i], "test %s: item %d", t.Name(), i)
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
t.Logf("m: %v", m)
|
||||
t.Logf("records: %v", testRecorder.Records())
|
||||
}
|
||||
testRecorder.Reset()
|
||||
}
|
||||
|
||||
runCount := 2 // we can run these tests even hundreds times to see its stability
|
||||
t.Run("1/1", func(t *testing.T) {
|
||||
for i := 0; i < runCount; i++ {
|
||||
test(t, setting.QueueSettings{BatchLength: 1, MaxWorkers: 1})
|
||||
}
|
||||
})
|
||||
t.Run("3/1", func(t *testing.T) {
|
||||
for i := 0; i < runCount; i++ {
|
||||
test(t, setting.QueueSettings{BatchLength: 3, MaxWorkers: 1})
|
||||
}
|
||||
})
|
||||
t.Run("4/5", func(t *testing.T) {
|
||||
for i := 0; i < runCount; i++ {
|
||||
test(t, setting.QueueSettings{BatchLength: 4, MaxWorkers: 5})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestWorkerPoolQueuePersistence(t *testing.T) {
|
||||
runCount := 2 // we can run these tests even hundreds times to see its stability
|
||||
t.Run("1/1", func(t *testing.T) {
|
||||
for i := 0; i < runCount; i++ {
|
||||
testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 1, MaxWorkers: 1, Length: 100})
|
||||
}
|
||||
})
|
||||
t.Run("3/1", func(t *testing.T) {
|
||||
for i := 0; i < runCount; i++ {
|
||||
testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 3, MaxWorkers: 1, Length: 100})
|
||||
}
|
||||
})
|
||||
t.Run("4/5", func(t *testing.T) {
|
||||
for i := 0; i < runCount; i++ {
|
||||
testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 4, MaxWorkers: 5, Length: 100})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func testWorkerPoolQueuePersistence(t *testing.T, queueSetting setting.QueueSettings) {
|
||||
testCount := queueSetting.Length
|
||||
queueSetting.Type = "level"
|
||||
queueSetting.Datadir = t.TempDir() + "/test-queue"
|
||||
|
||||
mu := sync.Mutex{}
|
||||
|
||||
var tasksQ1, tasksQ2 []string
|
||||
q1 := func() {
|
||||
startWhenAllReady := make(chan struct{}) // only start data consuming when the "testCount" tasks are all pushed into queue
|
||||
stopAt20Shutdown := make(chan struct{}) // stop and shutdown at the 20th item
|
||||
|
||||
testHandler := func(data ...string) []string {
|
||||
<-startWhenAllReady
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
for _, s := range data {
|
||||
mu.Lock()
|
||||
tasksQ1 = append(tasksQ1, s)
|
||||
mu.Unlock()
|
||||
|
||||
if s == "task-20" {
|
||||
close(stopAt20Shutdown)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
q, _ := NewWorkerPoolQueueBySetting("pr_patch_checker_test", queueSetting, testHandler, true)
|
||||
stop := runWorkerPoolQueue(q)
|
||||
for i := 0; i < testCount; i++ {
|
||||
_ = q.Push("task-" + strconv.Itoa(i))
|
||||
}
|
||||
close(startWhenAllReady)
|
||||
<-stopAt20Shutdown // it's possible to have more than 20 tasks executed
|
||||
stop()
|
||||
}
|
||||
|
||||
q1() // run some tasks and shutdown at an intermediate point
|
||||
|
||||
time.Sleep(100 * time.Millisecond) // because the handler in q1 has a slight delay, we need to wait for it to finish
|
||||
|
||||
q2 := func() {
|
||||
testHandler := func(data ...string) []string {
|
||||
for _, s := range data {
|
||||
mu.Lock()
|
||||
tasksQ2 = append(tasksQ2, s)
|
||||
mu.Unlock()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
q, _ := NewWorkerPoolQueueBySetting("pr_patch_checker_test", queueSetting, testHandler, true)
|
||||
stop := runWorkerPoolQueue(q)
|
||||
assert.NoError(t, q.FlushWithContext(context.Background(), 0))
|
||||
stop()
|
||||
}
|
||||
|
||||
q2() // restart the queue to continue to execute the tasks in it
|
||||
|
||||
assert.NotZero(t, len(tasksQ1))
|
||||
assert.NotZero(t, len(tasksQ2))
|
||||
assert.EqualValues(t, testCount, len(tasksQ1)+len(tasksQ2))
|
||||
}
|
||||
|
||||
func TestWorkerPoolQueueActiveWorkers(t *testing.T) {
|
||||
oldWorkerIdleDuration := workerIdleDuration
|
||||
workerIdleDuration = 300 * time.Millisecond
|
||||
defer func() {
|
||||
workerIdleDuration = oldWorkerIdleDuration
|
||||
}()
|
||||
|
||||
handler := func(items ...int) (unhandled []int) {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
return nil
|
||||
}
|
||||
|
||||
q, _ := NewWorkerPoolQueueBySetting("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 1, Length: 100}, handler, false)
|
||||
stop := runWorkerPoolQueue(q)
|
||||
for i := 0; i < 5; i++ {
|
||||
assert.NoError(t, q.Push(i))
|
||||
}
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
assert.EqualValues(t, 1, q.GetWorkerNumber())
|
||||
assert.EqualValues(t, 1, q.GetWorkerActiveNumber())
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
assert.EqualValues(t, 1, q.GetWorkerNumber())
|
||||
assert.EqualValues(t, 0, q.GetWorkerActiveNumber())
|
||||
time.Sleep(workerIdleDuration)
|
||||
assert.EqualValues(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working
|
||||
stop()
|
||||
|
||||
q, _ = NewWorkerPoolQueueBySetting("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 3, Length: 100}, handler, false)
|
||||
stop = runWorkerPoolQueue(q)
|
||||
for i := 0; i < 15; i++ {
|
||||
assert.NoError(t, q.Push(i))
|
||||
}
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
assert.EqualValues(t, 3, q.GetWorkerNumber())
|
||||
assert.EqualValues(t, 3, q.GetWorkerActiveNumber())
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
assert.EqualValues(t, 3, q.GetWorkerNumber())
|
||||
assert.EqualValues(t, 0, q.GetWorkerActiveNumber())
|
||||
time.Sleep(workerIdleDuration)
|
||||
assert.EqualValues(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working
|
||||
stop()
|
||||
}
|
||||
|
||||
func TestWorkerPoolQueueShutdown(t *testing.T) {
|
||||
oldUnhandledItemRequeueDuration := unhandledItemRequeueDuration.Load()
|
||||
unhandledItemRequeueDuration.Store(int64(100 * time.Millisecond))
|
||||
defer unhandledItemRequeueDuration.Store(oldUnhandledItemRequeueDuration)
|
||||
|
||||
// simulate a slow handler, it doesn't handle any item (all items will be pushed back to the queue)
|
||||
handlerCalled := make(chan struct{})
|
||||
handler := func(items ...int) (unhandled []int) {
|
||||
if items[0] == 0 {
|
||||
close(handlerCalled)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
return items
|
||||
}
|
||||
|
||||
qs := setting.QueueSettings{Type: "level", Datadir: t.TempDir() + "/queue", BatchLength: 3, MaxWorkers: 4, Length: 20}
|
||||
q, _ := NewWorkerPoolQueueBySetting("test-workpoolqueue", qs, handler, false)
|
||||
stop := runWorkerPoolQueue(q)
|
||||
for i := 0; i < qs.Length; i++ {
|
||||
assert.NoError(t, q.Push(i))
|
||||
}
|
||||
<-handlerCalled
|
||||
time.Sleep(50 * time.Millisecond) // wait for a while to make sure all workers are active
|
||||
assert.EqualValues(t, 4, q.GetWorkerActiveNumber())
|
||||
stop() // stop triggers shutdown
|
||||
assert.EqualValues(t, 0, q.GetWorkerActiveNumber())
|
||||
|
||||
// no item was ever handled, so we still get all of them again
|
||||
q, _ = NewWorkerPoolQueueBySetting("test-workpoolqueue", qs, handler, false)
|
||||
assert.EqualValues(t, 20, q.GetQueueItemNumber())
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue