Mikescher
/
SimpleCloudNotifier
1
0
Fork 0
Code Pull Requests Actions Releases Activity

Use multiple DB connections but retry failed requests

This commit is contained in:
Mike Schwörer 2022-12-20 09:52:33 +01:00
parent 00d77e508d
commit f7675be834
Signed by: Mikescher
GPG Key ID: D3C7172E0A70F8CF
5 changed files with 87 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
server/common/ginresp/wrapper.go
View File

@ -1,25 +1,80 @@
package ginresp
import "github.com/gin-gonic/gin"
import (
scn "blackforestbytes.com/simplecloudnotifier"
"github.com/gin-gonic/gin"
"github.com/mattn/go-sqlite3"
"github.com/rs/zerolog/log"
"gogs.mikescher.com/BlackForestBytes/goext/dataext"
"time"
)
type WHandlerFunc func(*gin.Context) HTTPResponse
func Wrap(fn WHandlerFunc) gin.HandlerFunc {
maxRetry := scn.Conf.RequestMaxRetry
retrySleep := scn.Conf.RequestRetrySleep
return func(g *gin.Context) {
reqctx := g.Request.Context()
wrap := fn(g)
if g.Writer.Written() {
panic("Writing in WrapperFunc is not supported")
if g.Request.Body != nil {
g.Request.Body = dataext.NewBufferedReadCloser(g.Request.Body)
}
if reqctx.Err() == nil {
wrap.Write(g)
for ctr := 1; ; ctr++ {
wrap := fn(g)
if g.Writer.Written() {
panic("Writing in WrapperFunc is not supported")
}
if ctr < maxRetry && isSqlite3Busy(wrap) {
log.Warn().Int("counter", ctr).Str("url", g.Request.URL.String()).Msg("Retry request (ErrBusy)")
err := resetBody(g)
if err != nil {
panic(err)
}
time.Sleep(retrySleep)
continue
}
if reqctx.Err() == nil {
wrap.Write(g)
}
return
}
}
}
func resetBody(g *gin.Context) error {
if g.Request.Body == nil {
return nil
}
err := g.Request.Body.(dataext.BufferedReadCloser).Reset()
if err != nil {
return err
}
return nil
}
func isSqlite3Busy(r HTTPResponse) bool {
if errwrap, ok := r.(*errorHTTPResponse); ok && errwrap != nil {
if s3err, ok := (errwrap.error).(sqlite3.Error); ok {
if s3err.Code == sqlite3.ErrBusy {
return true
}
}
}
return false
}

22
server/config.go
View File

@ -26,6 +26,8 @@ type Config struct {
DBCheckForeignKeys bool `env:"SCN_DB_CHECKFOREIGNKEYS"`
DBSingleConn bool `env:"SCN_DB_SINGLECONNECTION"`
RequestTimeout time.Duration `env:"SCN_REQUEST_TIMEOUT"`
RequestMaxRetry int `env:"SCN_REQUEST_MAXRETRY"`
RequestRetrySleep time.Duration `env:"SCN_REQUEST_RETRYSLEEP"`
ReturnRawErrors bool `env:"SCN_ERROR_RETURN"`
DummyFirebase bool `env:"SCN_DUMMY_FB"`
DummyGoogleAPI bool `env:"SCN_DUMMY_GOOG"`
@ -57,12 +59,14 @@ var configLocHost = func() Config {
DBJournal: "WAL",
DBTimeout: 5 * time.Second,
DBCheckForeignKeys: false,
DBSingleConn: true,
DBSingleConn: false,
DBMaxOpenConns: 5,
DBMaxIdleConns: 5,
DBConnMaxLifetime: 60 * time.Minute,
DBConnMaxIdleTime: 60 * time.Minute,
RequestTimeout: 16 * time.Second,
RequestMaxRetry: 8,
RequestRetrySleep: 100 * time.Millisecond,
ReturnRawErrors: true,
DummyFirebase: true,
FirebaseTokenURI: "",
@ -93,12 +97,14 @@ var configLocDocker = func() Config {
DBJournal: "WAL",
DBTimeout: 5 * time.Second,
DBCheckForeignKeys: false,
DBSingleConn: true,
DBSingleConn: false,
DBMaxOpenConns: 5,
DBMaxIdleConns: 5,
DBConnMaxLifetime: 60 * time.Minute,
DBConnMaxIdleTime: 60 * time.Minute,
RequestTimeout: 16 * time.Second,
RequestMaxRetry: 8,
RequestRetrySleep: 100 * time.Millisecond,
ReturnRawErrors: true,
DummyFirebase: true,
FirebaseTokenURI: "",
@ -129,12 +135,14 @@ var configDev = func() Config {
DBJournal: "WAL",
DBTimeout: 5 * time.Second,
DBCheckForeignKeys: false,
DBSingleConn: true,
DBSingleConn: false,
DBMaxOpenConns: 5,
DBMaxIdleConns: 5,
DBConnMaxLifetime: 60 * time.Minute,
DBConnMaxIdleTime: 60 * time.Minute,
RequestTimeout: 16 * time.Second,
RequestMaxRetry: 8,
RequestRetrySleep: 100 * time.Millisecond,
ReturnRawErrors: true,
DummyFirebase: false,
FirebaseTokenURI: "https://oauth2.googleapis.com/token",
@ -165,12 +173,14 @@ var configStag = func() Config {
DBJournal: "WAL",
DBTimeout: 5 * time.Second,
DBCheckForeignKeys: false,
DBSingleConn: true,
DBSingleConn: false,
DBMaxOpenConns: 5,
DBMaxIdleConns: 5,
DBConnMaxLifetime: 60 * time.Minute,
DBConnMaxIdleTime: 60 * time.Minute,
RequestTimeout: 16 * time.Second,
RequestMaxRetry: 8,
RequestRetrySleep: 100 * time.Millisecond,
ReturnRawErrors: true,
DummyFirebase: false,
FirebaseTokenURI: "https://oauth2.googleapis.com/token",
@ -201,12 +211,14 @@ var configProd = func() Config {
DBJournal: "WAL",
DBTimeout: 5 * time.Second,
DBCheckForeignKeys: false,
DBSingleConn: true,
DBSingleConn: false,
DBMaxOpenConns: 5,
DBMaxIdleConns: 5,
DBConnMaxLifetime: 60 * time.Minute,
DBConnMaxIdleTime: 60 * time.Minute,
RequestTimeout: 16 * time.Second,
RequestMaxRetry: 8,
RequestRetrySleep: 100 * time.Millisecond,
ReturnRawErrors: false,
DummyFirebase: false,
FirebaseTokenURI: "https://oauth2.googleapis.com/token",

2
server/go.mod
View File

@ -8,7 +8,7 @@ require (
github.com/mattn/go-sqlite3 v1.14.16
github.com/rs/zerolog v1.28.0
github.com/swaggo/swag v1.8.7
gogs.mikescher.com/BlackForestBytes/goext v0.0.36
gogs.mikescher.com/BlackForestBytes/goext v0.0.37
)
require (

2
server/go.sum
View File

@ -110,6 +110,8 @@ gogs.mikescher.com/BlackForestBytes/goext v0.0.35 h1:K5IMnAns68D6DmkryCN8CrLcmlo
gogs.mikescher.com/BlackForestBytes/goext v0.0.35/go.mod h1:/u9JtMwCP68ix4R9BJ/MT0Lm+QScmqIoyYZFKBGzv9g=
gogs.mikescher.com/BlackForestBytes/goext v0.0.36 h1:iOUYz2NEiObCCdBnkt8DPi1N8gH5H9q6qyJQpWp36rA=
gogs.mikescher.com/BlackForestBytes/goext v0.0.36/go.mod h1:/u9JtMwCP68ix4R9BJ/MT0Lm+QScmqIoyYZFKBGzv9g=
gogs.mikescher.com/BlackForestBytes/goext v0.0.37 h1:6XP5UOqiougzH0Xtzs5tIU4c0sAXmdMPCvGhRxqVwLU=
gogs.mikescher.com/BlackForestBytes/goext v0.0.37/go.mod h1:/u9JtMwCP68ix4R9BJ/MT0Lm+QScmqIoyYZFKBGzv9g=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI=
golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=

6
server/test/util/webserver.go
View File

@ -59,11 +59,15 @@ func StartSimpleWebserver(t *testing.T) (*logic.Application, string, func()) {
DBConnMaxLifetime: 1 * time.Second,
DBConnMaxIdleTime: 1 * time.Second,
RequestTimeout: 30 * time.Second,
RequestMaxRetry: 32,
RequestRetrySleep: 100 * time.Millisecond,
ReturnRawErrors: true,
DummyFirebase: true,
DBSingleConn: true,
DBSingleConn: false,
}
scn.Conf = conf
sqlite, err := db.NewDatabase(conf)
if err != nil {
TestFailErr(t, err)