Use multiple DB connections but retry failed requests

server/common/ginresp/wrapper.go

@@ -1,25 +1,80 @@
 package ginresp
 
-import "github.com/gin-gonic/gin"
+import (
+	scn "blackforestbytes.com/simplecloudnotifier"
+	"github.com/gin-gonic/gin"
+	"github.com/mattn/go-sqlite3"
+	"github.com/rs/zerolog/log"
+	"gogs.mikescher.com/BlackForestBytes/goext/dataext"
+	"time"
+)
 
 type WHandlerFunc func(*gin.Context) HTTPResponse
 
 func Wrap(fn WHandlerFunc) gin.HandlerFunc {
 
+	maxRetry := scn.Conf.RequestMaxRetry
+	retrySleep := scn.Conf.RequestRetrySleep
+
 	return func(g *gin.Context) {
 
 		reqctx := g.Request.Context()
 
+		if g.Request.Body != nil {
+			g.Request.Body = dataext.NewBufferedReadCloser(g.Request.Body)
+		}
+
+		for ctr := 1; ; ctr++ {
+
 			wrap := fn(g)
 
 			if g.Writer.Written() {
 				panic("Writing in WrapperFunc is not supported")
 			}
 
+			if ctr < maxRetry && isSqlite3Busy(wrap) {
+				log.Warn().Int("counter", ctr).Str("url", g.Request.URL.String()).Msg("Retry request (ErrBusy)")
+
+				err := resetBody(g)
+				if err != nil {
+					panic(err)
+				}
+
+				time.Sleep(retrySleep)
+				continue
+			}
+
 			if reqctx.Err() == nil {
 				wrap.Write(g)
 			}
 
+			return
+		}
+
 	}
 
 }
+
+func resetBody(g *gin.Context) error {
+	if g.Request.Body == nil {
+		return nil
+	}
+
+	err := g.Request.Body.(dataext.BufferedReadCloser).Reset()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func isSqlite3Busy(r HTTPResponse) bool {
+	if errwrap, ok := r.(*errorHTTPResponse); ok && errwrap != nil {
+		if s3err, ok := (errwrap.error).(sqlite3.Error); ok {
+			if s3err.Code == sqlite3.ErrBusy {
+				return true
+			}
+		}
+	}
+	return false
+}

server/config.go

@@ -26,6 +26,8 @@ type Config struct {
 	DBCheckForeignKeys  bool          `env:"SCN_DB_CHECKFOREIGNKEYS"`
 	DBSingleConn        bool          `env:"SCN_DB_SINGLECONNECTION"`
 	RequestTimeout      time.Duration `env:"SCN_REQUEST_TIMEOUT"`
+	RequestMaxRetry     int           `env:"SCN_REQUEST_MAXRETRY"`
+	RequestRetrySleep   time.Duration `env:"SCN_REQUEST_RETRYSLEEP"`
 	ReturnRawErrors     bool          `env:"SCN_ERROR_RETURN"`
 	DummyFirebase       bool          `env:"SCN_DUMMY_FB"`
 	DummyGoogleAPI      bool          `env:"SCN_DUMMY_GOOG"`
@@ -57,12 +59,14 @@ var configLocHost = func() Config {
 		DBJournal:           "WAL",
 		DBTimeout:           5 * time.Second,
 		DBCheckForeignKeys:  false,
-		DBSingleConn:        true,
+		DBSingleConn:        false,
 		DBMaxOpenConns:      5,
 		DBMaxIdleConns:      5,
 		DBConnMaxLifetime:   60 * time.Minute,
 		DBConnMaxIdleTime:   60 * time.Minute,
 		RequestTimeout:      16 * time.Second,
+		RequestMaxRetry:     8,
+		RequestRetrySleep:   100 * time.Millisecond,
 		ReturnRawErrors:     true,
 		DummyFirebase:       true,
 		FirebaseTokenURI:    "",
@@ -93,12 +97,14 @@ var configLocDocker = func() Config {
 		DBJournal:           "WAL",
 		DBTimeout:           5 * time.Second,
 		DBCheckForeignKeys:  false,
-		DBSingleConn:        true,
+		DBSingleConn:        false,
 		DBMaxOpenConns:      5,
 		DBMaxIdleConns:      5,
 		DBConnMaxLifetime:   60 * time.Minute,
 		DBConnMaxIdleTime:   60 * time.Minute,
 		RequestTimeout:      16 * time.Second,
+		RequestMaxRetry:     8,
+		RequestRetrySleep:   100 * time.Millisecond,
 		ReturnRawErrors:     true,
 		DummyFirebase:       true,
 		FirebaseTokenURI:    "",
@@ -129,12 +135,14 @@ var configDev = func() Config {
 		DBJournal:           "WAL",
 		DBTimeout:           5 * time.Second,
 		DBCheckForeignKeys:  false,
-		DBSingleConn:        true,
+		DBSingleConn:        false,
 		DBMaxOpenConns:      5,
 		DBMaxIdleConns:      5,
 		DBConnMaxLifetime:   60 * time.Minute,
 		DBConnMaxIdleTime:   60 * time.Minute,
 		RequestTimeout:      16 * time.Second,
+		RequestMaxRetry:     8,
+		RequestRetrySleep:   100 * time.Millisecond,
 		ReturnRawErrors:     true,
 		DummyFirebase:       false,
 		FirebaseTokenURI:    "https://oauth2.googleapis.com/token",
@@ -165,12 +173,14 @@ var configStag = func() Config {
 		DBJournal:           "WAL",
 		DBTimeout:           5 * time.Second,
 		DBCheckForeignKeys:  false,
-		DBSingleConn:        true,
+		DBSingleConn:        false,
 		DBMaxOpenConns:      5,
 		DBMaxIdleConns:      5,
 		DBConnMaxLifetime:   60 * time.Minute,
 		DBConnMaxIdleTime:   60 * time.Minute,
 		RequestTimeout:      16 * time.Second,
+		RequestMaxRetry:     8,
+		RequestRetrySleep:   100 * time.Millisecond,
 		ReturnRawErrors:     true,
 		DummyFirebase:       false,
 		FirebaseTokenURI:    "https://oauth2.googleapis.com/token",
@@ -201,12 +211,14 @@ var configProd = func() Config {
 		DBJournal:           "WAL",
 		DBTimeout:           5 * time.Second,
 		DBCheckForeignKeys:  false,
-		DBSingleConn:        true,
+		DBSingleConn:        false,
 		DBMaxOpenConns:      5,
 		DBMaxIdleConns:      5,
 		DBConnMaxLifetime:   60 * time.Minute,
 		DBConnMaxIdleTime:   60 * time.Minute,
 		RequestTimeout:      16 * time.Second,
+		RequestMaxRetry:     8,
+		RequestRetrySleep:   100 * time.Millisecond,
 		ReturnRawErrors:     false,
 		DummyFirebase:       false,
 		FirebaseTokenURI:    "https://oauth2.googleapis.com/token",

server/go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/mattn/go-sqlite3 v1.14.16
 	github.com/rs/zerolog v1.28.0
 	github.com/swaggo/swag v1.8.7
-	gogs.mikescher.com/BlackForestBytes/goext v0.0.36
+	gogs.mikescher.com/BlackForestBytes/goext v0.0.37
 )
 
 require (

server/go.sum

@@ -110,6 +110,8 @@ gogs.mikescher.com/BlackForestBytes/goext v0.0.35 h1:K5IMnAns68D6DmkryCN8CrLcmlo
 gogs.mikescher.com/BlackForestBytes/goext v0.0.35/go.mod h1:/u9JtMwCP68ix4R9BJ/MT0Lm+QScmqIoyYZFKBGzv9g=
 gogs.mikescher.com/BlackForestBytes/goext v0.0.36 h1:iOUYz2NEiObCCdBnkt8DPi1N8gH5H9q6qyJQpWp36rA=
 gogs.mikescher.com/BlackForestBytes/goext v0.0.36/go.mod h1:/u9JtMwCP68ix4R9BJ/MT0Lm+QScmqIoyYZFKBGzv9g=
+gogs.mikescher.com/BlackForestBytes/goext v0.0.37 h1:6XP5UOqiougzH0Xtzs5tIU4c0sAXmdMPCvGhRxqVwLU=
+gogs.mikescher.com/BlackForestBytes/goext v0.0.37/go.mod h1:/u9JtMwCP68ix4R9BJ/MT0Lm+QScmqIoyYZFKBGzv9g=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI=
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s=

server/test/util/webserver.go

@@ -59,11 +59,15 @@ func StartSimpleWebserver(t *testing.T) (*logic.Application, string, func()) {
 		DBConnMaxLifetime: 1 * time.Second,
 		DBConnMaxIdleTime: 1 * time.Second,
 		RequestTimeout:    30 * time.Second,
+		RequestMaxRetry:   32,
+		RequestRetrySleep: 100 * time.Millisecond,
 		ReturnRawErrors:   true,
 		DummyFirebase:     true,
-		DBSingleConn:      true,
+		DBSingleConn:      false,
 	}
 
+	scn.Conf = conf
+
 	sqlite, err := db.NewDatabase(conf)
 	if err != nil {
 		TestFailErr(t, err)