From 00d77e508d057a182b60bdcc7c1a70a8d78c6053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20Schw=C3=B6rer?= Date: Tue, 20 Dec 2022 09:22:18 +0100 Subject: [PATCH] Fix TestSendParallel by using only a single DB connection see https://github.com/mattn/go-sqlite3/issues/274 see https://github.com/mattn/go-sqlite3/issues/209 see https://stackoverflow.com/questions/32479071/sqlite3-error-database-is-locked-in-golang --- server/README.md | 2 ++ server/common/ginext/gin.go | 4 +++- server/common/ginresp/resp.go | 16 ++++++++++++++-- server/config.go | 12 ++++++++++++ server/db/database.go | 12 ++++++++---- server/test/send_test.go | 2 -- server/test/util/webserver.go | 1 + 7 files changed, 40 insertions(+), 9 deletions(-) diff --git a/server/README.md b/server/README.md index 1d09bd3..2ba0c76 100644 --- a/server/README.md +++ b/server/README.md @@ -15,6 +15,8 @@ - deploy + - diff my currently used scnsend script vs the one in the docs here + ------------------------------------------------------------------------------------------------------------------------------- - in my script: use (backupname || hostname) for sendername diff --git a/server/common/ginext/gin.go b/server/common/ginext/gin.go index 8089df1..c41045d 100644 --- a/server/common/ginext/gin.go +++ b/server/common/ginext/gin.go @@ -13,7 +13,9 @@ func NewEngine(cfg scn.Config) *gin.Engine { engine.RedirectFixedPath = false engine.RedirectTrailingSlash = false - engine.Use(CorsMiddleware()) + if cfg.Cors { + engine.Use(CorsMiddleware()) + } if cfg.GinDebug { ginlogger := gin.Logger() diff --git a/server/common/ginresp/resp.go b/server/common/ginresp/resp.go index 3ad8d86..1c8022c 100644 --- a/server/common/ginresp/resp.go +++ b/server/common/ginresp/resp.go @@ -51,6 +51,16 @@ func (j dataHTTPResponse) Write(g *gin.Context) { g.Data(j.statusCode, j.contentType, j.data) } +type errorHTTPResponse struct { + statusCode int + data any + error error +} + +func (j errorHTTPResponse) Write(g *gin.Context) { + g.JSON(j.statusCode, j.data) +} + func Status(sc int) HTTPResponse { return &emptyHTTPResponse{statusCode: sc} } @@ -98,7 +108,7 @@ func createApiError(g *gin.Context, ident string, status int, errorid apierr.API Msg(fmt.Sprintf("[%s] %s", ident, msg)) if scn.Conf.ReturnRawErrors { - return &jsonHTTPResponse{ + return &errorHTTPResponse{ statusCode: status, data: apiError{ Success: false, @@ -108,9 +118,10 @@ func createApiError(g *gin.Context, ident string, status int, errorid apierr.API RawError: langext.Ptr(langext.Conditional(e == nil, "", fmt.Sprintf("%+v", e))), Trace: string(debug.Stack()), }, + error: e, } } else { - return &jsonHTTPResponse{ + return &errorHTTPResponse{ statusCode: status, data: apiError{ Success: false, @@ -118,6 +129,7 @@ func createApiError(g *gin.Context, ident string, status int, errorid apierr.API ErrorHighlight: int(highlight), Message: msg, }, + error: e, } } } diff --git a/server/config.go b/server/config.go index 8837ac0..10fc2a5 100644 --- a/server/config.go +++ b/server/config.go @@ -24,6 +24,7 @@ type Config struct { DBConnMaxLifetime time.Duration `env:"SCN_DB_CONNEXTIONMAXLIFETIME"` DBConnMaxIdleTime time.Duration `env:"SCN_DB_CONNEXTIONMAXIDLETIME"` DBCheckForeignKeys bool `env:"SCN_DB_CHECKFOREIGNKEYS"` + DBSingleConn bool `env:"SCN_DB_SINGLECONNECTION"` RequestTimeout time.Duration `env:"SCN_REQUEST_TIMEOUT"` ReturnRawErrors bool `env:"SCN_ERROR_RETURN"` DummyFirebase bool `env:"SCN_DUMMY_FB"` @@ -39,6 +40,7 @@ type Config struct { GoogleAPIPrivateKey string `env:"SCN_GOOG_PRIVATEKEY"` GooglePackageName string `env:"SCN_GOOG_PACKAGENAME"` GoogleProProductID string `env:"SCN_GOOG_PROPRODUCTID"` + Cors bool `env:"SCN_CORS"` } var Conf Config @@ -55,6 +57,7 @@ var configLocHost = func() Config { DBJournal: "WAL", DBTimeout: 5 * time.Second, DBCheckForeignKeys: false, + DBSingleConn: true, DBMaxOpenConns: 5, DBMaxIdleConns: 5, DBConnMaxLifetime: 60 * time.Minute, @@ -74,6 +77,7 @@ var configLocHost = func() Config { GoogleAPIPrivateKey: "", GooglePackageName: "", GoogleProProductID: "", + Cors: true, } } @@ -89,6 +93,7 @@ var configLocDocker = func() Config { DBJournal: "WAL", DBTimeout: 5 * time.Second, DBCheckForeignKeys: false, + DBSingleConn: true, DBMaxOpenConns: 5, DBMaxIdleConns: 5, DBConnMaxLifetime: 60 * time.Minute, @@ -108,6 +113,7 @@ var configLocDocker = func() Config { GoogleAPIPrivateKey: "", GooglePackageName: "", GoogleProProductID: "", + Cors: true, } } @@ -123,6 +129,7 @@ var configDev = func() Config { DBJournal: "WAL", DBTimeout: 5 * time.Second, DBCheckForeignKeys: false, + DBSingleConn: true, DBMaxOpenConns: 5, DBMaxIdleConns: 5, DBConnMaxLifetime: 60 * time.Minute, @@ -142,6 +149,7 @@ var configDev = func() Config { GoogleAPIPrivateKey: confEnv("SCN_GOOG_PRIVATEKEY"), GooglePackageName: confEnv("SCN_GOOG_PACKAGENAME"), GoogleProProductID: confEnv("SCN_GOOG_PROPRODUCTID"), + Cors: true, } } @@ -157,6 +165,7 @@ var configStag = func() Config { DBJournal: "WAL", DBTimeout: 5 * time.Second, DBCheckForeignKeys: false, + DBSingleConn: true, DBMaxOpenConns: 5, DBMaxIdleConns: 5, DBConnMaxLifetime: 60 * time.Minute, @@ -176,6 +185,7 @@ var configStag = func() Config { GoogleAPIPrivateKey: confEnv("SCN_GOOG_PRIVATEKEY"), GooglePackageName: confEnv("SCN_GOOG_PACKAGENAME"), GoogleProProductID: confEnv("SCN_GOOG_PROPRODUCTID"), + Cors: true, } } @@ -191,6 +201,7 @@ var configProd = func() Config { DBJournal: "WAL", DBTimeout: 5 * time.Second, DBCheckForeignKeys: false, + DBSingleConn: true, DBMaxOpenConns: 5, DBMaxIdleConns: 5, DBConnMaxLifetime: 60 * time.Minute, @@ -210,6 +221,7 @@ var configProd = func() Config { GoogleAPIPrivateKey: confEnv("SCN_SCN_GOOG_PRIVATEKEY"), GooglePackageName: confEnv("SCN_SCN_GOOG_PACKAGENAME"), GoogleProProductID: confEnv("SCN_SCN_GOOG_PROPRODUCTID"), + Cors: true, } } diff --git a/server/db/database.go b/server/db/database.go index 9dadd33..65817c8 100644 --- a/server/db/database.go +++ b/server/db/database.go @@ -27,10 +27,14 @@ func NewDatabase(conf server.Config) (*Database, error) { return nil, err } - xdb.SetMaxOpenConns(5) - xdb.SetMaxIdleConns(5) - xdb.SetConnMaxLifetime(60 * time.Minute) - xdb.SetConnMaxIdleTime(60 * time.Minute) + if conf.DBSingleConn { + xdb.SetMaxOpenConns(1) + } else { + xdb.SetMaxOpenConns(5) + xdb.SetMaxIdleConns(5) + xdb.SetConnMaxLifetime(60 * time.Minute) + xdb.SetConnMaxIdleTime(60 * time.Minute) + } qqdb := sq.NewDB(xdb) diff --git a/server/test/send_test.go b/server/test/send_test.go index 8b7889f..f9103fc 100644 --- a/server/test/send_test.go +++ b/server/test/send_test.go @@ -1429,8 +1429,6 @@ func TestQuotaExceededPro(t *testing.T) { } func TestSendParallel(t *testing.T) { - t.SkipNow() - _, baseUrl, stop := tt.StartSimpleWebserver(t) defer stop() diff --git a/server/test/util/webserver.go b/server/test/util/webserver.go index 59cd5fa..48c9379 100644 --- a/server/test/util/webserver.go +++ b/server/test/util/webserver.go @@ -61,6 +61,7 @@ func StartSimpleWebserver(t *testing.T) (*logic.Application, string, func()) { RequestTimeout: 30 * time.Second, ReturnRawErrors: true, DummyFirebase: true, + DBSingleConn: true, } sqlite, err := db.NewDatabase(conf)