From 352f1ca0d1df93173e47e5a5440b38c2d1aede93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20Schw=C3=B6rer?= Date: Fri, 20 Sep 2024 17:13:36 +0200 Subject: [PATCH] Fully switch away from mattn sqlite to glebarez sqlite --- scnserver/Makefile | 2 +- scnserver/TODO.md | 2 - scnserver/api/handler/apiSenderNames.go | 38 +++++------ scnserver/api/handler/common.go | 28 +++----- scnserver/db/database.go | 1 + scnserver/db/impl/logs/database.go | 14 ++++ scnserver/db/impl/primary/database.go | 14 ++++ scnserver/db/impl/requests/database.go | 14 ++++ scnserver/go.mod | 3 +- scnserver/go.sum | 4 -- scnserver/logic/request.go | 17 +++-- scnserver/models/enums_gen.go | 2 +- scnserver/models/ids_gen.go | 2 +- scnserver/swagger/swagger.json | 91 +++++++++++++++++++++++-- scnserver/swagger/swagger.yaml | 66 ++++++++++++++++-- 15 files changed, 235 insertions(+), 63 deletions(-) diff --git a/scnserver/Makefile b/scnserver/Makefile index 9a48b2c..0c0a889 100644 --- a/scnserver/Makefile +++ b/scnserver/Makefile @@ -5,7 +5,7 @@ PORT=9090 NAMESPACE=$(shell git rev-parse --abbrev-ref HEAD) HASH=$(shell git rev-parse HEAD) -TAGS="timetzdata sqlite_fts5 sqlite_foreign_keys" +TAGS="timetzdata" .PHONY: test swagger pygmentize docker migrate dgi pygmentize lint docker diff --git a/scnserver/TODO.md b/scnserver/TODO.md index c281a65..c21348b 100644 --- a/scnserver/TODO.md +++ b/scnserver/TODO.md @@ -12,8 +12,6 @@ - exerr.New | exerr.Wrap - - Fork go-sqlite with always-on fts5 - #### UNSURE - (?) default-priority for channels diff --git a/scnserver/api/handler/apiSenderNames.go b/scnserver/api/handler/apiSenderNames.go index 5c54c8d..6bbb584 100644 --- a/scnserver/api/handler/apiSenderNames.go +++ b/scnserver/api/handler/apiSenderNames.go @@ -11,19 +11,19 @@ import ( // ListUserSenderNames swaggerdoc // -// @Summary List sender-names (of allthe messages of this user) -// @ID api-usersendernames-list -// @Tags API-v2 +// @Summary List sender-names (of allthe messages of this user) +// @ID api-usersendernames-list +// @Tags API-v2 // -// @Param uid path string true "UserID" +// @Param uid path string true "UserID" // -// @Success 200 {object} handler.ListUserKeys.response -// @Failure 400 {object} ginresp.apiError "supplied values/parameters cannot be parsed / are invalid" -// @Failure 401 {object} ginresp.apiError "user is not authorized / has missing permissions" -// @Failure 404 {object} ginresp.apiError "message not found" -// @Failure 500 {object} ginresp.apiError "internal server error" +// @Success 200 {object} handler.ListUserKeys.response +// @Failure 400 {object} ginresp.apiError "supplied values/parameters cannot be parsed / are invalid" +// @Failure 401 {object} ginresp.apiError "user is not authorized / has missing permissions" +// @Failure 404 {object} ginresp.apiError "message not found" +// @Failure 500 {object} ginresp.apiError "internal server error" // -// @Router /api/v2/users/{uid}/keys [GET] +// @Router /api/v2/users/{uid}/keys [GET] func (h APIHandler) ListUserSenderNames(pctx ginext.PreContext) ginext.HTTPResponse { type uri struct { UserID models.UserID `uri:"uid" binding:"entityid"` @@ -57,17 +57,17 @@ func (h APIHandler) ListUserSenderNames(pctx ginext.PreContext) ginext.HTTPRespo // ListSenderNames swaggerdoc // -// @Summary List sender-names (of all messages this user can view, eitehr own or foreign-subscribed) -// @ID api-sendernames-list -// @Tags API-v2 +// @Summary List sender-names (of all messages this user can view, eitehr own or foreign-subscribed) +// @ID api-sendernames-list +// @Tags API-v2 // -// @Success 200 {object} handler.ListSenderNames.response -// @Failure 400 {object} ginresp.apiError "supplied values/parameters cannot be parsed / are invalid" -// @Failure 401 {object} ginresp.apiError "user is not authorized / has missing permissions" -// @Failure 404 {object} ginresp.apiError "message not found" -// @Failure 500 {object} ginresp.apiError "internal server error" +// @Success 200 {object} handler.ListSenderNames.response +// @Failure 400 {object} ginresp.apiError "supplied values/parameters cannot be parsed / are invalid" +// @Failure 401 {object} ginresp.apiError "user is not authorized / has missing permissions" +// @Failure 404 {object} ginresp.apiError "message not found" +// @Failure 500 {object} ginresp.apiError "internal server error" // -// @Router /api/v2/sender-names [GET] +// @Router /api/v2/sender-names [GET] func (h APIHandler) ListSenderNames(pctx ginext.PreContext) ginext.HTTPResponse { type response struct { SenderNames []models.SenderNameStatistics `json:"sender_names"` diff --git a/scnserver/api/handler/common.go b/scnserver/api/handler/common.go index 68e6f34..06d3ae4 100644 --- a/scnserver/api/handler/common.go +++ b/scnserver/api/handler/common.go @@ -9,7 +9,6 @@ import ( "bytes" "errors" "github.com/gin-gonic/gin" - "github.com/mattn/go-sqlite3" "gogs.mikescher.com/BlackForestBytes/goext/ginext" "gogs.mikescher.com/BlackForestBytes/goext/langext" "gogs.mikescher.com/BlackForestBytes/goext/timeext" @@ -91,10 +90,9 @@ func (h CommonHandler) Ping(pctx ginext.PreContext) ginext.HTTPResponse { // @Router /api/db-test [post] func (h CommonHandler) DatabaseTest(pctx ginext.PreContext) ginext.HTTPResponse { type response struct { - Success bool `json:"success"` - LibVersion string `json:"libVersion"` - LibVersionNumber int `json:"libVersionNumber"` - SourceID string `json:"sourceID"` + Success bool `json:"success"` + LibVersion string `json:"libVersion"` + SourceID string `json:"sourceID"` } ctx, g, errResp := pctx.Start() @@ -105,18 +103,20 @@ func (h CommonHandler) DatabaseTest(pctx ginext.PreContext) ginext.HTTPResponse return h.app.DoRequest(ctx, g, models.TLockRead, func(ctx *logic.AppContext, finishSuccess func(r ginext.HTTPResponse) ginext.HTTPResponse) ginext.HTTPResponse { - libVersion, libVersionNumber, sourceID := sqlite3.Version() + versionStr, sourceID, err := h.app.Database.Primary.Version(ctx) + if err != nil { + return ginresp.InternalError(err) + } - err := h.app.Database.Ping(ctx) + err = h.app.Database.Ping(ctx) if err != nil { return ginresp.InternalError(err) } return ginext.JSON(http.StatusOK, response{ - Success: true, - LibVersion: libVersion, - LibVersionNumber: libVersionNumber, - SourceID: sourceID, + Success: true, + LibVersion: versionStr, + SourceID: sourceID, }) }) @@ -145,12 +145,6 @@ func (h CommonHandler) Health(pctx ginext.PreContext) ginext.HTTPResponse { return h.app.DoRequest(ctx, g, models.TLockReadWrite, func(ctx *logic.AppContext, finishSuccess func(r ginext.HTTPResponse) ginext.HTTPResponse) ginext.HTTPResponse { - _, libVersionNumber, _ := sqlite3.Version() - - if libVersionNumber < 3039000 { - return ginresp.InternalError(errors.New("sqlite version too low")) - } - tctx := simplectx.CreateSimpleContext(ctx, nil) err := h.app.Database.Ping(tctx) diff --git a/scnserver/db/database.go b/scnserver/db/database.go index 77cdc5a..a7185a8 100644 --- a/scnserver/db/database.go +++ b/scnserver/db/database.go @@ -10,6 +10,7 @@ type DatabaseImpl interface { Migrate(ctx context.Context) error Ping(ctx context.Context) error + Version(ctx context.Context) (string, string, error) BeginTx(ctx context.Context) (sq.Tx, error) Stop(ctx context.Context) error diff --git a/scnserver/db/impl/logs/database.go b/scnserver/db/impl/logs/database.go index 1f8de60..01c61c1 100644 --- a/scnserver/db/impl/logs/database.go +++ b/scnserver/db/impl/logs/database.go @@ -280,6 +280,20 @@ func (db *Database) Ping(ctx context.Context) error { return db.db.Ping(ctx) } +func (db *Database) Version(ctx context.Context) (string, string, error) { + type rt struct { + Version string `db:"version"` + SourceID string `db:"sourceID"` + } + + resp, err := sq.QuerySingle[rt](ctx, db.db, "SELECT sqlite_version() AS version, sqlite_source_id() AS sourceID", sq.PP{}, sq.SModeFast, sq.Safe) + if err != nil { + return "", "", err + } + + return resp.Version, resp.SourceID, nil +} + func (db *Database) BeginTx(ctx context.Context) (sq.Tx, error) { return db.db.BeginTransaction(ctx, sql.LevelDefault) } diff --git a/scnserver/db/impl/primary/database.go b/scnserver/db/impl/primary/database.go index d5669d1..c41c6b4 100644 --- a/scnserver/db/impl/primary/database.go +++ b/scnserver/db/impl/primary/database.go @@ -280,6 +280,20 @@ func (db *Database) Ping(ctx context.Context) error { return db.db.Ping(ctx) } +func (db *Database) Version(ctx context.Context) (string, string, error) { + type rt struct { + Version string `db:"version"` + SourceID string `db:"sourceID"` + } + + resp, err := sq.QuerySingle[rt](ctx, db.db, "SELECT sqlite_version() AS version, sqlite_source_id() AS sourceID", sq.PP{}, sq.SModeFast, sq.Safe) + if err != nil { + return "", "", err + } + + return resp.Version, resp.SourceID, nil +} + func (db *Database) BeginTx(ctx context.Context) (sq.Tx, error) { return db.db.BeginTransaction(ctx, sql.LevelDefault) } diff --git a/scnserver/db/impl/requests/database.go b/scnserver/db/impl/requests/database.go index 669b914..60f3821 100644 --- a/scnserver/db/impl/requests/database.go +++ b/scnserver/db/impl/requests/database.go @@ -280,6 +280,20 @@ func (db *Database) Ping(ctx context.Context) error { return db.db.Ping(ctx) } +func (db *Database) Version(ctx context.Context) (string, string, error) { + type rt struct { + Version string `db:"version"` + SourceID string `db:"sourceID"` + } + + resp, err := sq.QuerySingle[rt](ctx, db.db, "SELECT sqlite_version() AS version, sqlite_source_id() AS sourceID", sq.PP{}, sq.SModeFast, sq.Safe) + if err != nil { + return "", "", err + } + + return resp.Version, resp.SourceID, nil +} + func (db *Database) BeginTx(ctx context.Context) (sq.Tx, error) { return db.db.BeginTransaction(ctx, sql.LevelDefault) } diff --git a/scnserver/go.mod b/scnserver/go.mod index 3d6387a..84f3e84 100644 --- a/scnserver/go.mod +++ b/scnserver/go.mod @@ -10,8 +10,8 @@ require ( github.com/go-playground/validator/v10 v10.22.1 github.com/go-sql-driver/mysql v1.8.1 github.com/jmoiron/sqlx v1.4.0 - github.com/mattn/go-sqlite3 v1.14.22 github.com/rs/zerolog v1.33.0 + github.com/viney-shih/go-lock v1.1.2 gogs.mikescher.com/BlackForestBytes/goext v0.0.513 gopkg.in/loremipsum.v1 v1.1.2 ) @@ -44,7 +44,6 @@ require ( github.com/rs/xid v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect - github.com/viney-shih/go-lock v1.1.2 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect diff --git a/scnserver/go.sum b/scnserver/go.sum index a40e8c2..8b37e66 100644 --- a/scnserver/go.sum +++ b/scnserver/go.sum @@ -114,10 +114,6 @@ github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfS github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver v1.16.1 h1:rIVLL3q0IHM39dvE+z2ulZLp9ENZKThVfuvN/IiN4l8= go.mongodb.org/mongo-driver v1.16.1/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= -gogs.mikescher.com/BlackForestBytes/goext v0.0.511 h1:vAEhXdexKlLTNf/mGHzemp/4rzmv7n2jf5l4NK38tIw= -gogs.mikescher.com/BlackForestBytes/goext v0.0.511/go.mod h1:9Q9EjraeE3yih7EXgBlnwLLJXWuRZNsl7s5TVTh3aOU= -gogs.mikescher.com/BlackForestBytes/goext v0.0.512 h1:cdLUi1bSnGujtx8/K0fPql142aOvUyNPt+8aWMKKDFk= -gogs.mikescher.com/BlackForestBytes/goext v0.0.512/go.mod h1:9Q9EjraeE3yih7EXgBlnwLLJXWuRZNsl7s5TVTh3aOU= gogs.mikescher.com/BlackForestBytes/goext v0.0.513 h1:zGb5n220AYNElzQs611RYXfZlnUw6/VJJesfLftphkQ= gogs.mikescher.com/BlackForestBytes/goext v0.0.513/go.mod h1:9Q9EjraeE3yih7EXgBlnwLLJXWuRZNsl7s5TVTh3aOU= golang.org/x/arch v0.10.0 h1:S3huipmSclq3PJMNe76NGwkBR504WFkQ5dhzWzP8ZW8= diff --git a/scnserver/logic/request.go b/scnserver/logic/request.go index 8ab2c14..801eaf3 100644 --- a/scnserver/logic/request.go +++ b/scnserver/logic/request.go @@ -9,7 +9,7 @@ import ( "errors" "fmt" "github.com/gin-gonic/gin" - "github.com/mattn/go-sqlite3" + "github.com/glebarez/go-sqlite" "github.com/rs/zerolog/log" "gogs.mikescher.com/BlackForestBytes/goext/dataext" "gogs.mikescher.com/BlackForestBytes/goext/exerr" @@ -232,9 +232,18 @@ func isSqlite3Busy(r ginext.HTTPResponse) bool { if errwrap, ok := r.(interface{ Unwrap() error }); ok && errwrap != nil { orig := exerr.OriginalError(errwrap.Unwrap()) - var sqlite3Err sqlite3.Error - if errors.As(orig, &sqlite3Err) { - if sqlite3Err.Code == 5 { // [5] == SQLITE_BUSY + var sqliteErr *sqlite.Error + if errors.As(orig, &sqliteErr) { + if sqliteErr.Code() == 5 { // [5] == SQLITE_BUSY + return true + } + if sqliteErr.Code() == 517 { // [517] == SQLITE_BUSY_SNAPSHOT + return true + } + if sqliteErr.Code() == 261 { // [261] == SQLITE_BUSY_RECOVERY + return true + } + if sqliteErr.Code() == 773 { // [773] == SQLITE_BUSY_TIMEOUT return true } } diff --git a/scnserver/models/enums_gen.go b/scnserver/models/enums_gen.go index 7b992d6..cc38368 100644 --- a/scnserver/models/enums_gen.go +++ b/scnserver/models/enums_gen.go @@ -5,7 +5,7 @@ package models import "gogs.mikescher.com/BlackForestBytes/goext/langext" import "gogs.mikescher.com/BlackForestBytes/goext/enums" -const ChecksumEnumGenerator = "15ecd56622673b1af76a69e81c124d58f234e669c27e413241fe8cf7c3e7597c" // GoExtVersion: 0.0.513 +const ChecksumEnumGenerator = "a1b9c4807e1cec4ea2a8b19cd447aa4b47c13f8058a12470dff8eeec895ad8f8" // GoExtVersion: 0.0.513 // ================================ ClientType ================================ // diff --git a/scnserver/models/ids_gen.go b/scnserver/models/ids_gen.go index 27d1e21..45ddcd9 100644 --- a/scnserver/models/ids_gen.go +++ b/scnserver/models/ids_gen.go @@ -15,7 +15,7 @@ import "reflect" import "regexp" import "strings" -const ChecksumCharsetIDGenerator = "15ecd56622673b1af76a69e81c124d58f234e669c27e413241fe8cf7c3e7597c" // GoExtVersion: 0.0.513 +const ChecksumCharsetIDGenerator = "a1b9c4807e1cec4ea2a8b19cd447aa4b47c13f8058a12470dff8eeec895ad8f8" // GoExtVersion: 0.0.513 const idlen = 24 diff --git a/scnserver/swagger/swagger.json b/scnserver/swagger/swagger.json index 1046609..fca78ae 100644 --- a/scnserver/swagger/swagger.json +++ b/scnserver/swagger/swagger.json @@ -886,6 +886,11 @@ "name": "filter", "in": "query" }, + { + "type": "boolean", + "name": "has_sender", + "in": "query" + }, { "type": "string", "name": "next_page_token", @@ -1207,6 +1212,47 @@ } } }, + "/api/v2/sender-names": { + "get": { + "tags": [ + "API-v2" + ], + "summary": "List sender-names (of all messages this user can view, eitehr own or foreign-subscribed)", + "operationId": "api-sendernames-list", + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/handler.ListSenderNames.response" + } + }, + "400": { + "description": "supplied values/parameters cannot be parsed / are invalid", + "schema": { + "$ref": "#/definitions/ginresp.apiError" + } + }, + "401": { + "description": "user is not authorized / has missing permissions", + "schema": { + "$ref": "#/definitions/ginresp.apiError" + } + }, + "404": { + "description": "message not found", + "schema": { + "$ref": "#/definitions/ginresp.apiError" + } + }, + "500": { + "description": "internal server error", + "schema": { + "$ref": "#/definitions/ginresp.apiError" + } + } + } + } + }, "/api/v2/users": { "post": { "tags": [ @@ -2026,12 +2072,11 @@ }, "/api/v2/users/{uid}/keys": { "get": { - "description": "The request must be done with an ADMIN key, the returned keys are without their token.", "tags": [ "API-v2" ], - "summary": "List keys of the user", - "operationId": "api-tokenkeys-list", + "summary": "List sender-names (of allthe messages of this user)", + "operationId": "api-usersendernames-list", "parameters": [ { "type": "string", @@ -3344,9 +3389,6 @@ "libVersion": { "type": "string" }, - "libVersionNumber": { - "type": "integer" - }, "sourceID": { "type": "string" }, @@ -3423,6 +3465,9 @@ }, "page_size": { "type": "integer" + }, + "total_count": { + "type": "integer" } } }, @@ -3473,6 +3518,20 @@ }, "page_size": { "type": "integer" + }, + "total_count": { + "type": "integer" + } + } + }, + "handler.ListSenderNames.response": { + "type": "object", + "properties": { + "sender_names": { + "type": "array", + "items": { + "$ref": "#/definitions/models.SenderNameStatistics" + } } } }, @@ -3830,12 +3889,15 @@ "type": "string" }, "description_name": { + "description": "= DescriptionName, (optional), longer description text, initally nil", "type": "string" }, "display_name": { + "description": "= DisplayName, used for display purposes, can be changed, initially equals InternalName", "type": "string" }, "internal_name": { + "description": "= InternalName, used for sending, normalized, cannot be changed", "type": "string" }, "messages_sent": { @@ -4043,6 +4105,23 @@ } } }, + "models.SenderNameStatistics": { + "type": "object", + "properties": { + "count": { + "type": "integer" + }, + "first_timestamp": { + "type": "string" + }, + "last_timestamp": { + "type": "string" + }, + "name": { + "type": "string" + } + } + }, "models.Subscription": { "type": "object", "properties": { diff --git a/scnserver/swagger/swagger.yaml b/scnserver/swagger/swagger.yaml index ccaaba1..18cc1e0 100644 --- a/scnserver/swagger/swagger.yaml +++ b/scnserver/swagger/swagger.yaml @@ -198,8 +198,6 @@ definitions: properties: libVersion: type: string - libVersionNumber: - type: integer sourceID: type: string success: @@ -250,6 +248,8 @@ definitions: type: string page_size: type: integer + total_count: + type: integer type: object handler.ListChannelSubscriptions.response: properties: @@ -282,6 +282,15 @@ definitions: type: string page_size: type: integer + total_count: + type: integer + type: object + handler.ListSenderNames.response: + properties: + sender_names: + items: + $ref: '#/definitions/models.SenderNameStatistics' + type: array type: object handler.ListUserKeys.response: properties: @@ -517,10 +526,15 @@ definitions: channel_id: type: string description_name: + description: = DescriptionName, (optional), longer description text, initally + nil type: string display_name: + description: = DisplayName, used for display purposes, can be changed, initially + equals InternalName type: string internal_name: + description: = InternalName, used for sending, normalized, cannot be changed type: string messages_sent: type: integer @@ -661,6 +675,17 @@ definitions: usr_message_id: type: string type: object + models.SenderNameStatistics: + properties: + count: + type: integer + first_timestamp: + type: string + last_timestamp: + type: string + name: + type: string + type: object models.Subscription: properties: channel_id: @@ -1396,6 +1421,9 @@ paths: - in: query name: filter type: string + - in: query + name: has_sender + type: boolean - in: query name: next_page_token type: string @@ -1616,6 +1644,34 @@ paths: and only returns a subset of fields) tags: - API-v2 + /api/v2/sender-names: + get: + operationId: api-sendernames-list + responses: + "200": + description: OK + schema: + $ref: '#/definitions/handler.ListSenderNames.response' + "400": + description: supplied values/parameters cannot be parsed / are invalid + schema: + $ref: '#/definitions/ginresp.apiError' + "401": + description: user is not authorized / has missing permissions + schema: + $ref: '#/definitions/ginresp.apiError' + "404": + description: message not found + schema: + $ref: '#/definitions/ginresp.apiError' + "500": + description: internal server error + schema: + $ref: '#/definitions/ginresp.apiError' + summary: List sender-names (of all messages this user can view, eitehr own or + foreign-subscribed) + tags: + - API-v2 /api/v2/users: post: operationId: api-user-create @@ -2171,9 +2227,7 @@ paths: - API-v2 /api/v2/users/{uid}/keys: get: - description: The request must be done with an ADMIN key, the returned keys are - without their token. - operationId: api-tokenkeys-list + operationId: api-usersendernames-list parameters: - description: UserID in: path @@ -2201,7 +2255,7 @@ paths: description: internal server error schema: $ref: '#/definitions/ginresp.apiError' - summary: List keys of the user + summary: List sender-names (of allthe messages of this user) tags: - API-v2 post: