From f7675be834932420744820f104e14d15b898749d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20Schw=C3=B6rer?= Date: Tue, 20 Dec 2022 09:52:33 +0100 Subject: [PATCH] Use multiple DB connections but retry failed requests --- server/common/ginresp/wrapper.go | 69 ++++++++++++++++++++++++++++---- server/config.go | 22 +++++++--- server/go.mod | 2 +- server/go.sum | 2 + server/test/util/webserver.go | 6 ++- 5 files changed, 87 insertions(+), 14 deletions(-) diff --git a/server/common/ginresp/wrapper.go b/server/common/ginresp/wrapper.go index e4c4ad2..0089c77 100644 --- a/server/common/ginresp/wrapper.go +++ b/server/common/ginresp/wrapper.go @@ -1,25 +1,80 @@ package ginresp -import "github.com/gin-gonic/gin" +import ( + scn "blackforestbytes.com/simplecloudnotifier" + "github.com/gin-gonic/gin" + "github.com/mattn/go-sqlite3" + "github.com/rs/zerolog/log" + "gogs.mikescher.com/BlackForestBytes/goext/dataext" + "time" +) type WHandlerFunc func(*gin.Context) HTTPResponse func Wrap(fn WHandlerFunc) gin.HandlerFunc { + maxRetry := scn.Conf.RequestMaxRetry + retrySleep := scn.Conf.RequestRetrySleep + return func(g *gin.Context) { reqctx := g.Request.Context() - wrap := fn(g) - - if g.Writer.Written() { - panic("Writing in WrapperFunc is not supported") + if g.Request.Body != nil { + g.Request.Body = dataext.NewBufferedReadCloser(g.Request.Body) } - if reqctx.Err() == nil { - wrap.Write(g) + for ctr := 1; ; ctr++ { + + wrap := fn(g) + + if g.Writer.Written() { + panic("Writing in WrapperFunc is not supported") + } + + if ctr < maxRetry && isSqlite3Busy(wrap) { + log.Warn().Int("counter", ctr).Str("url", g.Request.URL.String()).Msg("Retry request (ErrBusy)") + + err := resetBody(g) + if err != nil { + panic(err) + } + + time.Sleep(retrySleep) + continue + } + + if reqctx.Err() == nil { + wrap.Write(g) + } + + return } } } + +func resetBody(g *gin.Context) error { + if g.Request.Body == nil { + return nil + } + + err := g.Request.Body.(dataext.BufferedReadCloser).Reset() + if err != nil { + return err + } + + return nil +} + +func isSqlite3Busy(r HTTPResponse) bool { + if errwrap, ok := r.(*errorHTTPResponse); ok && errwrap != nil { + if s3err, ok := (errwrap.error).(sqlite3.Error); ok { + if s3err.Code == sqlite3.ErrBusy { + return true + } + } + } + return false +} diff --git a/server/config.go b/server/config.go index 10fc2a5..63e7758 100644 --- a/server/config.go +++ b/server/config.go @@ -26,6 +26,8 @@ type Config struct { DBCheckForeignKeys bool `env:"SCN_DB_CHECKFOREIGNKEYS"` DBSingleConn bool `env:"SCN_DB_SINGLECONNECTION"` RequestTimeout time.Duration `env:"SCN_REQUEST_TIMEOUT"` + RequestMaxRetry int `env:"SCN_REQUEST_MAXRETRY"` + RequestRetrySleep time.Duration `env:"SCN_REQUEST_RETRYSLEEP"` ReturnRawErrors bool `env:"SCN_ERROR_RETURN"` DummyFirebase bool `env:"SCN_DUMMY_FB"` DummyGoogleAPI bool `env:"SCN_DUMMY_GOOG"` @@ -57,12 +59,14 @@ var configLocHost = func() Config { DBJournal: "WAL", DBTimeout: 5 * time.Second, DBCheckForeignKeys: false, - DBSingleConn: true, + DBSingleConn: false, DBMaxOpenConns: 5, DBMaxIdleConns: 5, DBConnMaxLifetime: 60 * time.Minute, DBConnMaxIdleTime: 60 * time.Minute, RequestTimeout: 16 * time.Second, + RequestMaxRetry: 8, + RequestRetrySleep: 100 * time.Millisecond, ReturnRawErrors: true, DummyFirebase: true, FirebaseTokenURI: "", @@ -93,12 +97,14 @@ var configLocDocker = func() Config { DBJournal: "WAL", DBTimeout: 5 * time.Second, DBCheckForeignKeys: false, - DBSingleConn: true, + DBSingleConn: false, DBMaxOpenConns: 5, DBMaxIdleConns: 5, DBConnMaxLifetime: 60 * time.Minute, DBConnMaxIdleTime: 60 * time.Minute, RequestTimeout: 16 * time.Second, + RequestMaxRetry: 8, + RequestRetrySleep: 100 * time.Millisecond, ReturnRawErrors: true, DummyFirebase: true, FirebaseTokenURI: "", @@ -129,12 +135,14 @@ var configDev = func() Config { DBJournal: "WAL", DBTimeout: 5 * time.Second, DBCheckForeignKeys: false, - DBSingleConn: true, + DBSingleConn: false, DBMaxOpenConns: 5, DBMaxIdleConns: 5, DBConnMaxLifetime: 60 * time.Minute, DBConnMaxIdleTime: 60 * time.Minute, RequestTimeout: 16 * time.Second, + RequestMaxRetry: 8, + RequestRetrySleep: 100 * time.Millisecond, ReturnRawErrors: true, DummyFirebase: false, FirebaseTokenURI: "https://oauth2.googleapis.com/token", @@ -165,12 +173,14 @@ var configStag = func() Config { DBJournal: "WAL", DBTimeout: 5 * time.Second, DBCheckForeignKeys: false, - DBSingleConn: true, + DBSingleConn: false, DBMaxOpenConns: 5, DBMaxIdleConns: 5, DBConnMaxLifetime: 60 * time.Minute, DBConnMaxIdleTime: 60 * time.Minute, RequestTimeout: 16 * time.Second, + RequestMaxRetry: 8, + RequestRetrySleep: 100 * time.Millisecond, ReturnRawErrors: true, DummyFirebase: false, FirebaseTokenURI: "https://oauth2.googleapis.com/token", @@ -201,12 +211,14 @@ var configProd = func() Config { DBJournal: "WAL", DBTimeout: 5 * time.Second, DBCheckForeignKeys: false, - DBSingleConn: true, + DBSingleConn: false, DBMaxOpenConns: 5, DBMaxIdleConns: 5, DBConnMaxLifetime: 60 * time.Minute, DBConnMaxIdleTime: 60 * time.Minute, RequestTimeout: 16 * time.Second, + RequestMaxRetry: 8, + RequestRetrySleep: 100 * time.Millisecond, ReturnRawErrors: false, DummyFirebase: false, FirebaseTokenURI: "https://oauth2.googleapis.com/token", diff --git a/server/go.mod b/server/go.mod index 8b77d60..1d7391b 100644 --- a/server/go.mod +++ b/server/go.mod @@ -8,7 +8,7 @@ require ( github.com/mattn/go-sqlite3 v1.14.16 github.com/rs/zerolog v1.28.0 github.com/swaggo/swag v1.8.7 - gogs.mikescher.com/BlackForestBytes/goext v0.0.36 + gogs.mikescher.com/BlackForestBytes/goext v0.0.37 ) require ( diff --git a/server/go.sum b/server/go.sum index 80baced..179dc3f 100644 --- a/server/go.sum +++ b/server/go.sum @@ -110,6 +110,8 @@ gogs.mikescher.com/BlackForestBytes/goext v0.0.35 h1:K5IMnAns68D6DmkryCN8CrLcmlo gogs.mikescher.com/BlackForestBytes/goext v0.0.35/go.mod h1:/u9JtMwCP68ix4R9BJ/MT0Lm+QScmqIoyYZFKBGzv9g= gogs.mikescher.com/BlackForestBytes/goext v0.0.36 h1:iOUYz2NEiObCCdBnkt8DPi1N8gH5H9q6qyJQpWp36rA= gogs.mikescher.com/BlackForestBytes/goext v0.0.36/go.mod h1:/u9JtMwCP68ix4R9BJ/MT0Lm+QScmqIoyYZFKBGzv9g= +gogs.mikescher.com/BlackForestBytes/goext v0.0.37 h1:6XP5UOqiougzH0Xtzs5tIU4c0sAXmdMPCvGhRxqVwLU= +gogs.mikescher.com/BlackForestBytes/goext v0.0.37/go.mod h1:/u9JtMwCP68ix4R9BJ/MT0Lm+QScmqIoyYZFKBGzv9g= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= diff --git a/server/test/util/webserver.go b/server/test/util/webserver.go index 48c9379..1a9e534 100644 --- a/server/test/util/webserver.go +++ b/server/test/util/webserver.go @@ -59,11 +59,15 @@ func StartSimpleWebserver(t *testing.T) (*logic.Application, string, func()) { DBConnMaxLifetime: 1 * time.Second, DBConnMaxIdleTime: 1 * time.Second, RequestTimeout: 30 * time.Second, + RequestMaxRetry: 32, + RequestRetrySleep: 100 * time.Millisecond, ReturnRawErrors: true, DummyFirebase: true, - DBSingleConn: true, + DBSingleConn: false, } + scn.Conf = conf + sqlite, err := db.NewDatabase(conf) if err != nil { TestFailErr(t, err)