Timeout on flush in testing (#16864)

* Timeout on flush in testing

At the end of each test the queues are flushed. At present there is no limit on the
length of time a flush can take which can lead to long flushes.

However, if the CI task is cancelled we lose the log information as to where the long
flush was taking place.

This PR simply adds a default time limit of 2 minutes - at which point an error will
be produced. This should allow us to more easily find the culprit.

Signed-off-by: Andrew Thornton <art27@cantab.net>

* return better error

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: 6543 <6543@obermui.de>
This commit is contained in:
zeripath 2021-08-30 05:27:51 +01:00 committed by GitHub
parent b0ff429882
commit 06b9d553bc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 13 additions and 2 deletions

View file

@ -121,7 +121,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() {
fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), slowFlush) fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), slowFlush)
} }
}) })
if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil { if err := queue.GetManager().FlushAll(context.Background(), 2*time.Minute); err != nil {
t.Errorf("Flushing queues failed with error %v", err) t.Errorf("Flushing queues failed with error %v", err)
} }
timer.Stop() timer.Stop()

View file

@ -9,6 +9,7 @@ import (
"fmt" "fmt"
"reflect" "reflect"
"sort" "sort"
"strings"
"sync" "sync"
"time" "time"
@ -169,7 +170,17 @@ func (m *Manager) FlushAll(baseCtx context.Context, timeout time.Duration) error
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
return ctx.Err() mqs := m.ManagedQueues()
nonEmptyQueues := []string{}
for _, mq := range mqs {
if !mq.IsEmpty() {
nonEmptyQueues = append(nonEmptyQueues, mq.Name)
}
}
if len(nonEmptyQueues) > 0 {
return fmt.Errorf("flush timeout with non-empty queues: %s", strings.Join(nonEmptyQueues, ", "))
}
return nil
default: default:
} }
mqs := m.ManagedQueues() mqs := m.ManagedQueues()