Fully switch away from mattn sqlite to glebarez sqlite

This commit is contained in:
Mike Schwörer 2024-09-20 17:13:36 +02:00
parent 584a9e983f
commit 352f1ca0d1
Signed by: Mikescher
GPG Key ID: D3C7172E0A70F8CF
15 changed files with 235 additions and 63 deletions

View File

@ -5,7 +5,7 @@ PORT=9090
NAMESPACE=$(shell git rev-parse --abbrev-ref HEAD) NAMESPACE=$(shell git rev-parse --abbrev-ref HEAD)
HASH=$(shell git rev-parse HEAD) HASH=$(shell git rev-parse HEAD)
TAGS="timetzdata sqlite_fts5 sqlite_foreign_keys" TAGS="timetzdata"
.PHONY: test swagger pygmentize docker migrate dgi pygmentize lint docker .PHONY: test swagger pygmentize docker migrate dgi pygmentize lint docker

View File

@ -12,8 +12,6 @@
- exerr.New | exerr.Wrap - exerr.New | exerr.Wrap
- Fork go-sqlite with always-on fts5
#### UNSURE #### UNSURE
- (?) default-priority for channels - (?) default-priority for channels

View File

@ -9,7 +9,6 @@ import (
"bytes" "bytes"
"errors" "errors"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/mattn/go-sqlite3"
"gogs.mikescher.com/BlackForestBytes/goext/ginext" "gogs.mikescher.com/BlackForestBytes/goext/ginext"
"gogs.mikescher.com/BlackForestBytes/goext/langext" "gogs.mikescher.com/BlackForestBytes/goext/langext"
"gogs.mikescher.com/BlackForestBytes/goext/timeext" "gogs.mikescher.com/BlackForestBytes/goext/timeext"
@ -93,7 +92,6 @@ func (h CommonHandler) DatabaseTest(pctx ginext.PreContext) ginext.HTTPResponse
type response struct { type response struct {
Success bool `json:"success"` Success bool `json:"success"`
LibVersion string `json:"libVersion"` LibVersion string `json:"libVersion"`
LibVersionNumber int `json:"libVersionNumber"`
SourceID string `json:"sourceID"` SourceID string `json:"sourceID"`
} }
@ -105,17 +103,19 @@ func (h CommonHandler) DatabaseTest(pctx ginext.PreContext) ginext.HTTPResponse
return h.app.DoRequest(ctx, g, models.TLockRead, func(ctx *logic.AppContext, finishSuccess func(r ginext.HTTPResponse) ginext.HTTPResponse) ginext.HTTPResponse { return h.app.DoRequest(ctx, g, models.TLockRead, func(ctx *logic.AppContext, finishSuccess func(r ginext.HTTPResponse) ginext.HTTPResponse) ginext.HTTPResponse {
libVersion, libVersionNumber, sourceID := sqlite3.Version() versionStr, sourceID, err := h.app.Database.Primary.Version(ctx)
if err != nil {
return ginresp.InternalError(err)
}
err := h.app.Database.Ping(ctx) err = h.app.Database.Ping(ctx)
if err != nil { if err != nil {
return ginresp.InternalError(err) return ginresp.InternalError(err)
} }
return ginext.JSON(http.StatusOK, response{ return ginext.JSON(http.StatusOK, response{
Success: true, Success: true,
LibVersion: libVersion, LibVersion: versionStr,
LibVersionNumber: libVersionNumber,
SourceID: sourceID, SourceID: sourceID,
}) })
@ -145,12 +145,6 @@ func (h CommonHandler) Health(pctx ginext.PreContext) ginext.HTTPResponse {
return h.app.DoRequest(ctx, g, models.TLockReadWrite, func(ctx *logic.AppContext, finishSuccess func(r ginext.HTTPResponse) ginext.HTTPResponse) ginext.HTTPResponse { return h.app.DoRequest(ctx, g, models.TLockReadWrite, func(ctx *logic.AppContext, finishSuccess func(r ginext.HTTPResponse) ginext.HTTPResponse) ginext.HTTPResponse {
_, libVersionNumber, _ := sqlite3.Version()
if libVersionNumber < 3039000 {
return ginresp.InternalError(errors.New("sqlite version too low"))
}
tctx := simplectx.CreateSimpleContext(ctx, nil) tctx := simplectx.CreateSimpleContext(ctx, nil)
err := h.app.Database.Ping(tctx) err := h.app.Database.Ping(tctx)

View File

@ -10,6 +10,7 @@ type DatabaseImpl interface {
Migrate(ctx context.Context) error Migrate(ctx context.Context) error
Ping(ctx context.Context) error Ping(ctx context.Context) error
Version(ctx context.Context) (string, string, error)
BeginTx(ctx context.Context) (sq.Tx, error) BeginTx(ctx context.Context) (sq.Tx, error)
Stop(ctx context.Context) error Stop(ctx context.Context) error

View File

@ -280,6 +280,20 @@ func (db *Database) Ping(ctx context.Context) error {
return db.db.Ping(ctx) return db.db.Ping(ctx)
} }
func (db *Database) Version(ctx context.Context) (string, string, error) {
type rt struct {
Version string `db:"version"`
SourceID string `db:"sourceID"`
}
resp, err := sq.QuerySingle[rt](ctx, db.db, "SELECT sqlite_version() AS version, sqlite_source_id() AS sourceID", sq.PP{}, sq.SModeFast, sq.Safe)
if err != nil {
return "", "", err
}
return resp.Version, resp.SourceID, nil
}
func (db *Database) BeginTx(ctx context.Context) (sq.Tx, error) { func (db *Database) BeginTx(ctx context.Context) (sq.Tx, error) {
return db.db.BeginTransaction(ctx, sql.LevelDefault) return db.db.BeginTransaction(ctx, sql.LevelDefault)
} }

View File

@ -280,6 +280,20 @@ func (db *Database) Ping(ctx context.Context) error {
return db.db.Ping(ctx) return db.db.Ping(ctx)
} }
func (db *Database) Version(ctx context.Context) (string, string, error) {
type rt struct {
Version string `db:"version"`
SourceID string `db:"sourceID"`
}
resp, err := sq.QuerySingle[rt](ctx, db.db, "SELECT sqlite_version() AS version, sqlite_source_id() AS sourceID", sq.PP{}, sq.SModeFast, sq.Safe)
if err != nil {
return "", "", err
}
return resp.Version, resp.SourceID, nil
}
func (db *Database) BeginTx(ctx context.Context) (sq.Tx, error) { func (db *Database) BeginTx(ctx context.Context) (sq.Tx, error) {
return db.db.BeginTransaction(ctx, sql.LevelDefault) return db.db.BeginTransaction(ctx, sql.LevelDefault)
} }

View File

@ -280,6 +280,20 @@ func (db *Database) Ping(ctx context.Context) error {
return db.db.Ping(ctx) return db.db.Ping(ctx)
} }
func (db *Database) Version(ctx context.Context) (string, string, error) {
type rt struct {
Version string `db:"version"`
SourceID string `db:"sourceID"`
}
resp, err := sq.QuerySingle[rt](ctx, db.db, "SELECT sqlite_version() AS version, sqlite_source_id() AS sourceID", sq.PP{}, sq.SModeFast, sq.Safe)
if err != nil {
return "", "", err
}
return resp.Version, resp.SourceID, nil
}
func (db *Database) BeginTx(ctx context.Context) (sq.Tx, error) { func (db *Database) BeginTx(ctx context.Context) (sq.Tx, error) {
return db.db.BeginTransaction(ctx, sql.LevelDefault) return db.db.BeginTransaction(ctx, sql.LevelDefault)
} }

View File

@ -10,8 +10,8 @@ require (
github.com/go-playground/validator/v10 v10.22.1 github.com/go-playground/validator/v10 v10.22.1
github.com/go-sql-driver/mysql v1.8.1 github.com/go-sql-driver/mysql v1.8.1
github.com/jmoiron/sqlx v1.4.0 github.com/jmoiron/sqlx v1.4.0
github.com/mattn/go-sqlite3 v1.14.22
github.com/rs/zerolog v1.33.0 github.com/rs/zerolog v1.33.0
github.com/viney-shih/go-lock v1.1.2
gogs.mikescher.com/BlackForestBytes/goext v0.0.513 gogs.mikescher.com/BlackForestBytes/goext v0.0.513
gopkg.in/loremipsum.v1 v1.1.2 gopkg.in/loremipsum.v1 v1.1.2
) )
@ -44,7 +44,6 @@ require (
github.com/rs/xid v1.6.0 // indirect github.com/rs/xid v1.6.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.2.12 // indirect github.com/ugorji/go/codec v1.2.12 // indirect
github.com/viney-shih/go-lock v1.1.2 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect

View File

@ -114,10 +114,6 @@ github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfS
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.mongodb.org/mongo-driver v1.16.1 h1:rIVLL3q0IHM39dvE+z2ulZLp9ENZKThVfuvN/IiN4l8= go.mongodb.org/mongo-driver v1.16.1 h1:rIVLL3q0IHM39dvE+z2ulZLp9ENZKThVfuvN/IiN4l8=
go.mongodb.org/mongo-driver v1.16.1/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= go.mongodb.org/mongo-driver v1.16.1/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw=
gogs.mikescher.com/BlackForestBytes/goext v0.0.511 h1:vAEhXdexKlLTNf/mGHzemp/4rzmv7n2jf5l4NK38tIw=
gogs.mikescher.com/BlackForestBytes/goext v0.0.511/go.mod h1:9Q9EjraeE3yih7EXgBlnwLLJXWuRZNsl7s5TVTh3aOU=
gogs.mikescher.com/BlackForestBytes/goext v0.0.512 h1:cdLUi1bSnGujtx8/K0fPql142aOvUyNPt+8aWMKKDFk=
gogs.mikescher.com/BlackForestBytes/goext v0.0.512/go.mod h1:9Q9EjraeE3yih7EXgBlnwLLJXWuRZNsl7s5TVTh3aOU=
gogs.mikescher.com/BlackForestBytes/goext v0.0.513 h1:zGb5n220AYNElzQs611RYXfZlnUw6/VJJesfLftphkQ= gogs.mikescher.com/BlackForestBytes/goext v0.0.513 h1:zGb5n220AYNElzQs611RYXfZlnUw6/VJJesfLftphkQ=
gogs.mikescher.com/BlackForestBytes/goext v0.0.513/go.mod h1:9Q9EjraeE3yih7EXgBlnwLLJXWuRZNsl7s5TVTh3aOU= gogs.mikescher.com/BlackForestBytes/goext v0.0.513/go.mod h1:9Q9EjraeE3yih7EXgBlnwLLJXWuRZNsl7s5TVTh3aOU=
golang.org/x/arch v0.10.0 h1:S3huipmSclq3PJMNe76NGwkBR504WFkQ5dhzWzP8ZW8= golang.org/x/arch v0.10.0 h1:S3huipmSclq3PJMNe76NGwkBR504WFkQ5dhzWzP8ZW8=

View File

@ -9,7 +9,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"github.com/gin-gonic/gin" "github.com/gin-gonic/gin"
"github.com/mattn/go-sqlite3" "github.com/glebarez/go-sqlite"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
"gogs.mikescher.com/BlackForestBytes/goext/dataext" "gogs.mikescher.com/BlackForestBytes/goext/dataext"
"gogs.mikescher.com/BlackForestBytes/goext/exerr" "gogs.mikescher.com/BlackForestBytes/goext/exerr"
@ -232,9 +232,18 @@ func isSqlite3Busy(r ginext.HTTPResponse) bool {
if errwrap, ok := r.(interface{ Unwrap() error }); ok && errwrap != nil { if errwrap, ok := r.(interface{ Unwrap() error }); ok && errwrap != nil {
orig := exerr.OriginalError(errwrap.Unwrap()) orig := exerr.OriginalError(errwrap.Unwrap())
var sqlite3Err sqlite3.Error var sqliteErr *sqlite.Error
if errors.As(orig, &sqlite3Err) { if errors.As(orig, &sqliteErr) {
if sqlite3Err.Code == 5 { // [5] == SQLITE_BUSY if sqliteErr.Code() == 5 { // [5] == SQLITE_BUSY
return true
}
if sqliteErr.Code() == 517 { // [517] == SQLITE_BUSY_SNAPSHOT
return true
}
if sqliteErr.Code() == 261 { // [261] == SQLITE_BUSY_RECOVERY
return true
}
if sqliteErr.Code() == 773 { // [773] == SQLITE_BUSY_TIMEOUT
return true return true
} }
} }

View File

@ -5,7 +5,7 @@ package models
import "gogs.mikescher.com/BlackForestBytes/goext/langext" import "gogs.mikescher.com/BlackForestBytes/goext/langext"
import "gogs.mikescher.com/BlackForestBytes/goext/enums" import "gogs.mikescher.com/BlackForestBytes/goext/enums"
const ChecksumEnumGenerator = "15ecd56622673b1af76a69e81c124d58f234e669c27e413241fe8cf7c3e7597c" // GoExtVersion: 0.0.513 const ChecksumEnumGenerator = "a1b9c4807e1cec4ea2a8b19cd447aa4b47c13f8058a12470dff8eeec895ad8f8" // GoExtVersion: 0.0.513
// ================================ ClientType ================================ // ================================ ClientType ================================
// //

View File

@ -15,7 +15,7 @@ import "reflect"
import "regexp" import "regexp"
import "strings" import "strings"
const ChecksumCharsetIDGenerator = "15ecd56622673b1af76a69e81c124d58f234e669c27e413241fe8cf7c3e7597c" // GoExtVersion: 0.0.513 const ChecksumCharsetIDGenerator = "a1b9c4807e1cec4ea2a8b19cd447aa4b47c13f8058a12470dff8eeec895ad8f8" // GoExtVersion: 0.0.513
const idlen = 24 const idlen = 24

View File

@ -886,6 +886,11 @@
"name": "filter", "name": "filter",
"in": "query" "in": "query"
}, },
{
"type": "boolean",
"name": "has_sender",
"in": "query"
},
{ {
"type": "string", "type": "string",
"name": "next_page_token", "name": "next_page_token",
@ -1207,6 +1212,47 @@
} }
} }
}, },
"/api/v2/sender-names": {
"get": {
"tags": [
"API-v2"
],
"summary": "List sender-names (of all messages this user can view, eitehr own or foreign-subscribed)",
"operationId": "api-sendernames-list",
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/handler.ListSenderNames.response"
}
},
"400": {
"description": "supplied values/parameters cannot be parsed / are invalid",
"schema": {
"$ref": "#/definitions/ginresp.apiError"
}
},
"401": {
"description": "user is not authorized / has missing permissions",
"schema": {
"$ref": "#/definitions/ginresp.apiError"
}
},
"404": {
"description": "message not found",
"schema": {
"$ref": "#/definitions/ginresp.apiError"
}
},
"500": {
"description": "internal server error",
"schema": {
"$ref": "#/definitions/ginresp.apiError"
}
}
}
}
},
"/api/v2/users": { "/api/v2/users": {
"post": { "post": {
"tags": [ "tags": [
@ -2026,12 +2072,11 @@
}, },
"/api/v2/users/{uid}/keys": { "/api/v2/users/{uid}/keys": {
"get": { "get": {
"description": "The request must be done with an ADMIN key, the returned keys are without their token.",
"tags": [ "tags": [
"API-v2" "API-v2"
], ],
"summary": "List keys of the user", "summary": "List sender-names (of allthe messages of this user)",
"operationId": "api-tokenkeys-list", "operationId": "api-usersendernames-list",
"parameters": [ "parameters": [
{ {
"type": "string", "type": "string",
@ -3344,9 +3389,6 @@
"libVersion": { "libVersion": {
"type": "string" "type": "string"
}, },
"libVersionNumber": {
"type": "integer"
},
"sourceID": { "sourceID": {
"type": "string" "type": "string"
}, },
@ -3423,6 +3465,9 @@
}, },
"page_size": { "page_size": {
"type": "integer" "type": "integer"
},
"total_count": {
"type": "integer"
} }
} }
}, },
@ -3473,6 +3518,20 @@
}, },
"page_size": { "page_size": {
"type": "integer" "type": "integer"
},
"total_count": {
"type": "integer"
}
}
},
"handler.ListSenderNames.response": {
"type": "object",
"properties": {
"sender_names": {
"type": "array",
"items": {
"$ref": "#/definitions/models.SenderNameStatistics"
}
} }
} }
}, },
@ -3830,12 +3889,15 @@
"type": "string" "type": "string"
}, },
"description_name": { "description_name": {
"description": "= DescriptionName, (optional), longer description text, initally nil",
"type": "string" "type": "string"
}, },
"display_name": { "display_name": {
"description": "= DisplayName, used for display purposes, can be changed, initially equals InternalName",
"type": "string" "type": "string"
}, },
"internal_name": { "internal_name": {
"description": "= InternalName, used for sending, normalized, cannot be changed",
"type": "string" "type": "string"
}, },
"messages_sent": { "messages_sent": {
@ -4043,6 +4105,23 @@
} }
} }
}, },
"models.SenderNameStatistics": {
"type": "object",
"properties": {
"count": {
"type": "integer"
},
"first_timestamp": {
"type": "string"
},
"last_timestamp": {
"type": "string"
},
"name": {
"type": "string"
}
}
},
"models.Subscription": { "models.Subscription": {
"type": "object", "type": "object",
"properties": { "properties": {

View File

@ -198,8 +198,6 @@ definitions:
properties: properties:
libVersion: libVersion:
type: string type: string
libVersionNumber:
type: integer
sourceID: sourceID:
type: string type: string
success: success:
@ -250,6 +248,8 @@ definitions:
type: string type: string
page_size: page_size:
type: integer type: integer
total_count:
type: integer
type: object type: object
handler.ListChannelSubscriptions.response: handler.ListChannelSubscriptions.response:
properties: properties:
@ -282,6 +282,15 @@ definitions:
type: string type: string
page_size: page_size:
type: integer type: integer
total_count:
type: integer
type: object
handler.ListSenderNames.response:
properties:
sender_names:
items:
$ref: '#/definitions/models.SenderNameStatistics'
type: array
type: object type: object
handler.ListUserKeys.response: handler.ListUserKeys.response:
properties: properties:
@ -517,10 +526,15 @@ definitions:
channel_id: channel_id:
type: string type: string
description_name: description_name:
description: = DescriptionName, (optional), longer description text, initally
nil
type: string type: string
display_name: display_name:
description: = DisplayName, used for display purposes, can be changed, initially
equals InternalName
type: string type: string
internal_name: internal_name:
description: = InternalName, used for sending, normalized, cannot be changed
type: string type: string
messages_sent: messages_sent:
type: integer type: integer
@ -661,6 +675,17 @@ definitions:
usr_message_id: usr_message_id:
type: string type: string
type: object type: object
models.SenderNameStatistics:
properties:
count:
type: integer
first_timestamp:
type: string
last_timestamp:
type: string
name:
type: string
type: object
models.Subscription: models.Subscription:
properties: properties:
channel_id: channel_id:
@ -1396,6 +1421,9 @@ paths:
- in: query - in: query
name: filter name: filter
type: string type: string
- in: query
name: has_sender
type: boolean
- in: query - in: query
name: next_page_token name: next_page_token
type: string type: string
@ -1616,6 +1644,34 @@ paths:
and only returns a subset of fields) and only returns a subset of fields)
tags: tags:
- API-v2 - API-v2
/api/v2/sender-names:
get:
operationId: api-sendernames-list
responses:
"200":
description: OK
schema:
$ref: '#/definitions/handler.ListSenderNames.response'
"400":
description: supplied values/parameters cannot be parsed / are invalid
schema:
$ref: '#/definitions/ginresp.apiError'
"401":
description: user is not authorized / has missing permissions
schema:
$ref: '#/definitions/ginresp.apiError'
"404":
description: message not found
schema:
$ref: '#/definitions/ginresp.apiError'
"500":
description: internal server error
schema:
$ref: '#/definitions/ginresp.apiError'
summary: List sender-names (of all messages this user can view, eitehr own or
foreign-subscribed)
tags:
- API-v2
/api/v2/users: /api/v2/users:
post: post:
operationId: api-user-create operationId: api-user-create
@ -2171,9 +2227,7 @@ paths:
- API-v2 - API-v2
/api/v2/users/{uid}/keys: /api/v2/users/{uid}/keys:
get: get:
description: The request must be done with an ADMIN key, the returned keys are operationId: api-usersendernames-list
without their token.
operationId: api-tokenkeys-list
parameters: parameters:
- description: UserID - description: UserID
in: path in: path
@ -2201,7 +2255,7 @@ paths:
description: internal server error description: internal server error
schema: schema:
$ref: '#/definitions/ginresp.apiError' $ref: '#/definitions/ginresp.apiError'
summary: List keys of the user summary: List sender-names (of allthe messages of this user)
tags: tags:
- API-v2 - API-v2
post: post: