Fully switch away from mattn sqlite to glebarez sqlite

scnserver/Makefile

@@ -5,7 +5,7 @@ PORT=9090
 NAMESPACE=$(shell git rev-parse --abbrev-ref HEAD)
 HASH=$(shell git rev-parse HEAD)
 
-TAGS="timetzdata sqlite_fts5 sqlite_foreign_keys"
+TAGS="timetzdata"
 
 .PHONY: test swagger pygmentize docker migrate dgi pygmentize lint docker
 

scnserver/TODO.md

@@ -12,8 +12,6 @@
 
  - exerr.New | exerr.Wrap
 
- - Fork go-sqlite with always-on fts5
-
 #### UNSURE
 
  - (?) default-priority for channels

scnserver/api/handler/apiSenderNames.go

@@ -11,19 +11,19 @@ import (
 
 // ListUserSenderNames swaggerdoc
 //
-//	@Summary		List sender-names (of allthe messages of this user)
+//	@Summary	List sender-names (of allthe messages of this user)
-//	@ID				api-usersendernames-list
+//	@ID			api-usersendernames-list
-//	@Tags			API-v2
+//	@Tags		API-v2
 //
-//	@Param			uid	path		string	true	"UserID"
+//	@Param		uid	path		string	true	"UserID"
 //
-//	@Success		200	{object}	handler.ListUserKeys.response
+//	@Success	200	{object}	handler.ListUserKeys.response
-//	@Failure		400	{object}	ginresp.apiError	"supplied values/parameters cannot be parsed / are invalid"
+//	@Failure	400	{object}	ginresp.apiError	"supplied values/parameters cannot be parsed / are invalid"
-//	@Failure		401	{object}	ginresp.apiError	"user is not authorized / has missing permissions"
+//	@Failure	401	{object}	ginresp.apiError	"user is not authorized / has missing permissions"
-//	@Failure		404	{object}	ginresp.apiError	"message not found"
+//	@Failure	404	{object}	ginresp.apiError	"message not found"
-//	@Failure		500	{object}	ginresp.apiError	"internal server error"
+//	@Failure	500	{object}	ginresp.apiError	"internal server error"
 //
-//	@Router			/api/v2/users/{uid}/keys [GET]
+//	@Router		/api/v2/users/{uid}/keys [GET]
 func (h APIHandler) ListUserSenderNames(pctx ginext.PreContext) ginext.HTTPResponse {
 	type uri struct {
 		UserID models.UserID `uri:"uid" binding:"entityid"`
@@ -57,17 +57,17 @@ func (h APIHandler) ListUserSenderNames(pctx ginext.PreContext) ginext.HTTPRespo
 
 // ListSenderNames swaggerdoc
 //
-//	@Summary		List sender-names (of all messages this user can view, eitehr own or foreign-subscribed)
+//	@Summary	List sender-names (of all messages this user can view, eitehr own or foreign-subscribed)
-//	@ID				api-sendernames-list
+//	@ID			api-sendernames-list
-//	@Tags			API-v2
+//	@Tags		API-v2
 //
-//	@Success		200	{object}	handler.ListSenderNames.response
+//	@Success	200	{object}	handler.ListSenderNames.response
-//	@Failure		400	{object}	ginresp.apiError	"supplied values/parameters cannot be parsed / are invalid"
+//	@Failure	400	{object}	ginresp.apiError	"supplied values/parameters cannot be parsed / are invalid"
-//	@Failure		401	{object}	ginresp.apiError	"user is not authorized / has missing permissions"
+//	@Failure	401	{object}	ginresp.apiError	"user is not authorized / has missing permissions"
-//	@Failure		404	{object}	ginresp.apiError	"message not found"
+//	@Failure	404	{object}	ginresp.apiError	"message not found"
-//	@Failure		500	{object}	ginresp.apiError	"internal server error"
+//	@Failure	500	{object}	ginresp.apiError	"internal server error"
 //
-//	@Router			/api/v2/sender-names [GET]
+//	@Router		/api/v2/sender-names [GET]
 func (h APIHandler) ListSenderNames(pctx ginext.PreContext) ginext.HTTPResponse {
 	type response struct {
 		SenderNames []models.SenderNameStatistics `json:"sender_names"`

scnserver/api/handler/common.go

@@ -9,7 +9,6 @@ import (
 	"bytes"
 	"errors"
 	"github.com/gin-gonic/gin"
-	"github.com/mattn/go-sqlite3"
 	"gogs.mikescher.com/BlackForestBytes/goext/ginext"
 	"gogs.mikescher.com/BlackForestBytes/goext/langext"
 	"gogs.mikescher.com/BlackForestBytes/goext/timeext"
@@ -91,10 +90,9 @@ func (h CommonHandler) Ping(pctx ginext.PreContext) ginext.HTTPResponse {
 //	@Router		/api/db-test [post]
 func (h CommonHandler) DatabaseTest(pctx ginext.PreContext) ginext.HTTPResponse {
 	type response struct {
-		Success          bool   `json:"success"`
+		Success    bool   `json:"success"`
-		LibVersion       string `json:"libVersion"`
+		LibVersion string `json:"libVersion"`
-		LibVersionNumber int    `json:"libVersionNumber"`
+		SourceID   string `json:"sourceID"`
-		SourceID         string `json:"sourceID"`
 	}
 
 	ctx, g, errResp := pctx.Start()
@@ -105,18 +103,20 @@ func (h CommonHandler) DatabaseTest(pctx ginext.PreContext) ginext.HTTPResponse
 
 	return h.app.DoRequest(ctx, g, models.TLockRead, func(ctx *logic.AppContext, finishSuccess func(r ginext.HTTPResponse) ginext.HTTPResponse) ginext.HTTPResponse {
 
-		libVersion, libVersionNumber, sourceID := sqlite3.Version()
+		versionStr, sourceID, err := h.app.Database.Primary.Version(ctx)
+		if err != nil {
+			return ginresp.InternalError(err)
+		}
 
-		err := h.app.Database.Ping(ctx)
+		err = h.app.Database.Ping(ctx)
 		if err != nil {
 			return ginresp.InternalError(err)
 		}
 
 		return ginext.JSON(http.StatusOK, response{
-			Success:          true,
+			Success:    true,
-			LibVersion:       libVersion,
+			LibVersion: versionStr,
-			LibVersionNumber: libVersionNumber,
+			SourceID:   sourceID,
-			SourceID:         sourceID,
 		})
 
 	})
@@ -145,12 +145,6 @@ func (h CommonHandler) Health(pctx ginext.PreContext) ginext.HTTPResponse {
 
 	return h.app.DoRequest(ctx, g, models.TLockReadWrite, func(ctx *logic.AppContext, finishSuccess func(r ginext.HTTPResponse) ginext.HTTPResponse) ginext.HTTPResponse {
 
-		_, libVersionNumber, _ := sqlite3.Version()
-
-		if libVersionNumber < 3039000 {
-			return ginresp.InternalError(errors.New("sqlite version too low"))
-		}
-
 		tctx := simplectx.CreateSimpleContext(ctx, nil)
 
 		err := h.app.Database.Ping(tctx)

scnserver/db/database.go

@@ -10,6 +10,7 @@ type DatabaseImpl interface {
 
 	Migrate(ctx context.Context) error
 	Ping(ctx context.Context) error
+	Version(ctx context.Context) (string, string, error)
 	BeginTx(ctx context.Context) (sq.Tx, error)
 	Stop(ctx context.Context) error
 

scnserver/db/impl/logs/database.go

@@ -280,6 +280,20 @@ func (db *Database) Ping(ctx context.Context) error {
 	return db.db.Ping(ctx)
 }
 
+func (db *Database) Version(ctx context.Context) (string, string, error) {
+	type rt struct {
+		Version  string `db:"version"`
+		SourceID string `db:"sourceID"`
+	}
+
+	resp, err := sq.QuerySingle[rt](ctx, db.db, "SELECT sqlite_version() AS version, sqlite_source_id() AS sourceID", sq.PP{}, sq.SModeFast, sq.Safe)
+	if err != nil {
+		return "", "", err
+	}
+
+	return resp.Version, resp.SourceID, nil
+}
+
 func (db *Database) BeginTx(ctx context.Context) (sq.Tx, error) {
 	return db.db.BeginTransaction(ctx, sql.LevelDefault)
 }

scnserver/db/impl/primary/database.go

@@ -280,6 +280,20 @@ func (db *Database) Ping(ctx context.Context) error {
 	return db.db.Ping(ctx)
 }
 
+func (db *Database) Version(ctx context.Context) (string, string, error) {
+	type rt struct {
+		Version  string `db:"version"`
+		SourceID string `db:"sourceID"`
+	}
+
+	resp, err := sq.QuerySingle[rt](ctx, db.db, "SELECT sqlite_version() AS version, sqlite_source_id() AS sourceID", sq.PP{}, sq.SModeFast, sq.Safe)
+	if err != nil {
+		return "", "", err
+	}
+
+	return resp.Version, resp.SourceID, nil
+}
+
 func (db *Database) BeginTx(ctx context.Context) (sq.Tx, error) {
 	return db.db.BeginTransaction(ctx, sql.LevelDefault)
 }

scnserver/db/impl/requests/database.go

@@ -280,6 +280,20 @@ func (db *Database) Ping(ctx context.Context) error {
 	return db.db.Ping(ctx)
 }
 
+func (db *Database) Version(ctx context.Context) (string, string, error) {
+	type rt struct {
+		Version  string `db:"version"`
+		SourceID string `db:"sourceID"`
+	}
+
+	resp, err := sq.QuerySingle[rt](ctx, db.db, "SELECT sqlite_version() AS version, sqlite_source_id() AS sourceID", sq.PP{}, sq.SModeFast, sq.Safe)
+	if err != nil {
+		return "", "", err
+	}
+
+	return resp.Version, resp.SourceID, nil
+}
+
 func (db *Database) BeginTx(ctx context.Context) (sq.Tx, error) {
 	return db.db.BeginTransaction(ctx, sql.LevelDefault)
 }

scnserver/go.mod

@@ -10,8 +10,8 @@ require (
 	github.com/go-playground/validator/v10 v10.22.1
 	github.com/go-sql-driver/mysql v1.8.1
 	github.com/jmoiron/sqlx v1.4.0
-	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/rs/zerolog v1.33.0
+	github.com/viney-shih/go-lock v1.1.2
 	gogs.mikescher.com/BlackForestBytes/goext v0.0.513
 	gopkg.in/loremipsum.v1 v1.1.2
 )
@@ -44,7 +44,6 @@ require (
 	github.com/rs/xid v1.6.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
-	github.com/viney-shih/go-lock v1.1.2 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect

scnserver/go.sum

@@ -114,10 +114,6 @@ github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfS
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.mongodb.org/mongo-driver v1.16.1 h1:rIVLL3q0IHM39dvE+z2ulZLp9ENZKThVfuvN/IiN4l8=
 go.mongodb.org/mongo-driver v1.16.1/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw=
-gogs.mikescher.com/BlackForestBytes/goext v0.0.511 h1:vAEhXdexKlLTNf/mGHzemp/4rzmv7n2jf5l4NK38tIw=
-gogs.mikescher.com/BlackForestBytes/goext v0.0.511/go.mod h1:9Q9EjraeE3yih7EXgBlnwLLJXWuRZNsl7s5TVTh3aOU=
-gogs.mikescher.com/BlackForestBytes/goext v0.0.512 h1:cdLUi1bSnGujtx8/K0fPql142aOvUyNPt+8aWMKKDFk=
-gogs.mikescher.com/BlackForestBytes/goext v0.0.512/go.mod h1:9Q9EjraeE3yih7EXgBlnwLLJXWuRZNsl7s5TVTh3aOU=
 gogs.mikescher.com/BlackForestBytes/goext v0.0.513 h1:zGb5n220AYNElzQs611RYXfZlnUw6/VJJesfLftphkQ=
 gogs.mikescher.com/BlackForestBytes/goext v0.0.513/go.mod h1:9Q9EjraeE3yih7EXgBlnwLLJXWuRZNsl7s5TVTh3aOU=
 golang.org/x/arch v0.10.0 h1:S3huipmSclq3PJMNe76NGwkBR504WFkQ5dhzWzP8ZW8=

scnserver/logic/request.go

@@ -9,7 +9,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/gin-gonic/gin"
-	"github.com/mattn/go-sqlite3"
+	"github.com/glebarez/go-sqlite"
 	"github.com/rs/zerolog/log"
 	"gogs.mikescher.com/BlackForestBytes/goext/dataext"
 	"gogs.mikescher.com/BlackForestBytes/goext/exerr"
@@ -232,9 +232,18 @@ func isSqlite3Busy(r ginext.HTTPResponse) bool {
 	if errwrap, ok := r.(interface{ Unwrap() error }); ok && errwrap != nil {
 		orig := exerr.OriginalError(errwrap.Unwrap())
 
-		var sqlite3Err sqlite3.Error
+		var sqliteErr *sqlite.Error
-		if errors.As(orig, &sqlite3Err) {
+		if errors.As(orig, &sqliteErr) {
-			if sqlite3Err.Code == 5 { // [5] == SQLITE_BUSY
+			if sqliteErr.Code() == 5 { // [5] == SQLITE_BUSY
+				return true
+			}
+			if sqliteErr.Code() == 517 { // [517] == SQLITE_BUSY_SNAPSHOT
+				return true
+			}
+			if sqliteErr.Code() == 261 { // [261] == SQLITE_BUSY_RECOVERY
+				return true
+			}
+			if sqliteErr.Code() == 773 { // [773] == SQLITE_BUSY_TIMEOUT
 				return true
 			}
 		}

scnserver/models/enums_gen.go

@@ -5,7 +5,7 @@ package models
 import "gogs.mikescher.com/BlackForestBytes/goext/langext"
 import "gogs.mikescher.com/BlackForestBytes/goext/enums"
 
-const ChecksumEnumGenerator = "15ecd56622673b1af76a69e81c124d58f234e669c27e413241fe8cf7c3e7597c" // GoExtVersion: 0.0.513
+const ChecksumEnumGenerator = "a1b9c4807e1cec4ea2a8b19cd447aa4b47c13f8058a12470dff8eeec895ad8f8" // GoExtVersion: 0.0.513
 
 // ================================ ClientType ================================
 //

scnserver/models/ids_gen.go

@@ -15,7 +15,7 @@ import "reflect"
 import "regexp"
 import "strings"
 
-const ChecksumCharsetIDGenerator = "15ecd56622673b1af76a69e81c124d58f234e669c27e413241fe8cf7c3e7597c" // GoExtVersion: 0.0.513
+const ChecksumCharsetIDGenerator = "a1b9c4807e1cec4ea2a8b19cd447aa4b47c13f8058a12470dff8eeec895ad8f8" // GoExtVersion: 0.0.513
 
 const idlen = 24
 

scnserver/swagger/swagger.json

@@ -886,6 +886,11 @@
                         "name": "filter",
                         "in": "query"
                     },
+                    {
+                        "type": "boolean",
+                        "name": "has_sender",
+                        "in": "query"
+                    },
                     {
                         "type": "string",
                         "name": "next_page_token",
@@ -1207,6 +1212,47 @@
                 }
             }
         },
+        "/api/v2/sender-names": {
+            "get": {
+                "tags": [
+                    "API-v2"
+                ],
+                "summary": "List sender-names (of all messages this user can view, eitehr own or foreign-subscribed)",
+                "operationId": "api-sendernames-list",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "$ref": "#/definitions/handler.ListSenderNames.response"
+                        }
+                    },
+                    "400": {
+                        "description": "supplied values/parameters cannot be parsed / are invalid",
+                        "schema": {
+                            "$ref": "#/definitions/ginresp.apiError"
+                        }
+                    },
+                    "401": {
+                        "description": "user is not authorized / has missing permissions",
+                        "schema": {
+                            "$ref": "#/definitions/ginresp.apiError"
+                        }
+                    },
+                    "404": {
+                        "description": "message not found",
+                        "schema": {
+                            "$ref": "#/definitions/ginresp.apiError"
+                        }
+                    },
+                    "500": {
+                        "description": "internal server error",
+                        "schema": {
+                            "$ref": "#/definitions/ginresp.apiError"
+                        }
+                    }
+                }
+            }
+        },
         "/api/v2/users": {
             "post": {
                 "tags": [
@@ -2026,12 +2072,11 @@
         },
         "/api/v2/users/{uid}/keys": {
             "get": {
-                "description": "The request must be done with an ADMIN key, the returned keys are without their token.",
                 "tags": [
                     "API-v2"
                 ],
-                "summary": "List keys of the user",
+                "summary": "List sender-names (of allthe messages of this user)",
-                "operationId": "api-tokenkeys-list",
+                "operationId": "api-usersendernames-list",
                 "parameters": [
                     {
                         "type": "string",
@@ -3344,9 +3389,6 @@
                 "libVersion": {
                     "type": "string"
                 },
-                "libVersionNumber": {
-                    "type": "integer"
-                },
                 "sourceID": {
                     "type": "string"
                 },
@@ -3423,6 +3465,9 @@
                 },
                 "page_size": {
                     "type": "integer"
+                },
+                "total_count": {
+                    "type": "integer"
                 }
             }
         },
@@ -3473,6 +3518,20 @@
                 },
                 "page_size": {
                     "type": "integer"
+                },
+                "total_count": {
+                    "type": "integer"
+                }
+            }
+        },
+        "handler.ListSenderNames.response": {
+            "type": "object",
+            "properties": {
+                "sender_names": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/models.SenderNameStatistics"
+                    }
                 }
             }
         },
@@ -3830,12 +3889,15 @@
                     "type": "string"
                 },
                 "description_name": {
+                    "description": "= DescriptionName, (optional), longer description text, initally nil",
                     "type": "string"
                 },
                 "display_name": {
+                    "description": "= DisplayName, used for display purposes, can be changed, initially equals InternalName",
                     "type": "string"
                 },
                 "internal_name": {
+                    "description": "= InternalName, used for sending, normalized, cannot be changed",
                     "type": "string"
                 },
                 "messages_sent": {
@@ -4043,6 +4105,23 @@
                 }
             }
         },
+        "models.SenderNameStatistics": {
+            "type": "object",
+            "properties": {
+                "count": {
+                    "type": "integer"
+                },
+                "first_timestamp": {
+                    "type": "string"
+                },
+                "last_timestamp": {
+                    "type": "string"
+                },
+                "name": {
+                    "type": "string"
+                }
+            }
+        },
         "models.Subscription": {
             "type": "object",
             "properties": {

scnserver/swagger/swagger.yaml

@@ -198,8 +198,6 @@ definitions:
     properties:
       libVersion:
         type: string
-      libVersionNumber:
-        type: integer
       sourceID:
         type: string
       success:
@@ -250,6 +248,8 @@ definitions:
         type: string
       page_size:
         type: integer
+      total_count:
+        type: integer
     type: object
   handler.ListChannelSubscriptions.response:
     properties:
@@ -282,6 +282,15 @@ definitions:
         type: string
       page_size:
         type: integer
+      total_count:
+        type: integer
+    type: object
+  handler.ListSenderNames.response:
+    properties:
+      sender_names:
+        items:
+          $ref: '#/definitions/models.SenderNameStatistics'
+        type: array
     type: object
   handler.ListUserKeys.response:
     properties:
@@ -517,10 +526,15 @@ definitions:
       channel_id:
         type: string
       description_name:
+        description: = DescriptionName, (optional), longer description text, initally
+          nil
         type: string
       display_name:
+        description: = DisplayName, used for display purposes, can be changed, initially
+          equals InternalName
         type: string
       internal_name:
+        description: = InternalName, used for sending, normalized, cannot be changed
         type: string
       messages_sent:
         type: integer
@@ -661,6 +675,17 @@ definitions:
       usr_message_id:
         type: string
     type: object
+  models.SenderNameStatistics:
+    properties:
+      count:
+        type: integer
+      first_timestamp:
+        type: string
+      last_timestamp:
+        type: string
+      name:
+        type: string
+    type: object
   models.Subscription:
     properties:
       channel_id:
@@ -1396,6 +1421,9 @@ paths:
       - in: query
         name: filter
         type: string
+      - in: query
+        name: has_sender
+        type: boolean
       - in: query
         name: next_page_token
         type: string
@@ -1616,6 +1644,34 @@ paths:
         and only returns a subset of fields)
       tags:
       - API-v2
+  /api/v2/sender-names:
+    get:
+      operationId: api-sendernames-list
+      responses:
+        "200":
+          description: OK
+          schema:
+            $ref: '#/definitions/handler.ListSenderNames.response'
+        "400":
+          description: supplied values/parameters cannot be parsed / are invalid
+          schema:
+            $ref: '#/definitions/ginresp.apiError'
+        "401":
+          description: user is not authorized / has missing permissions
+          schema:
+            $ref: '#/definitions/ginresp.apiError'
+        "404":
+          description: message not found
+          schema:
+            $ref: '#/definitions/ginresp.apiError'
+        "500":
+          description: internal server error
+          schema:
+            $ref: '#/definitions/ginresp.apiError'
+      summary: List sender-names (of all messages this user can view, eitehr own or
+        foreign-subscribed)
+      tags:
+      - API-v2
   /api/v2/users:
     post:
       operationId: api-user-create
@@ -2171,9 +2227,7 @@ paths:
       - API-v2
   /api/v2/users/{uid}/keys:
     get:
-      description: The request must be done with an ADMIN key, the returned keys are
+      operationId: api-usersendernames-list
-        without their token.
-      operationId: api-tokenkeys-list
       parameters:
       - description: UserID
         in: path
@@ -2201,7 +2255,7 @@ paths:
           description: internal server error
           schema:
             $ref: '#/definitions/ginresp.apiError'
-      summary: List keys of the user
+      summary: List sender-names (of allthe messages of this user)
       tags:
       - API-v2
     post: