From 7ddaf5d9aa7c847e0ab19cb5da498f1eb7423ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20Schw=C3=B6rer?= Date: Mon, 16 Sep 2024 18:26:28 +0200 Subject: [PATCH] Migrate deliveries.next_delivery from type:string to type:int (SCNTime) --- scnserver/db/impl/primary/database.go | 45 ++++ scnserver/db/impl/primary/deliveries.go | 7 +- scnserver/db/schema/assets.go | 9 +- scnserver/db/schema/primary_6.ddl | 234 ++++++++++++++++++ scnserver/db/schema/primary_migration_5_6.ddl | 49 ++++ scnserver/models/delivery.go | 22 +- 6 files changed, 349 insertions(+), 17 deletions(-) create mode 100644 scnserver/db/schema/primary_6.ddl create mode 100644 scnserver/db/schema/primary_migration_5_6.ddl diff --git a/scnserver/db/impl/primary/database.go b/scnserver/db/impl/primary/database.go index 0cd831c..a03a96a 100644 --- a/scnserver/db/impl/primary/database.go +++ b/scnserver/db/impl/primary/database.go @@ -241,6 +241,51 @@ func (db *Database) Migrate(outerctx context.Context) error { } else { log.Debug().Str("schemHash", schemHashDB).Msg("Verified Schema consistency (primary db)") } + + log.Info().Int("currschema", currschema).Msg("Upgrade schema from 5 -> 6") + + _, err = tx.Exec(tctx, schema.PrimaryMigration_5_6, sq.PP{}) + if err != nil { + return err + } + + currschema = 6 + + err = db.WriteMetaInt(tctx, "schema", int64(currschema)) + if err != nil { + return err + } + + err = db.WriteMetaString(tctx, "schema_hash", schema.PrimarySchema[currschema].Hash) + if err != nil { + return err + } + + log.Info().Int("currschema", currschema).Msg("Upgrade schema from 5 -> 6 succesfully") + + ppReInit = true + } + + if currschema == 6 { + + schemaHashMeta, err := db.ReadMetaString(tctx, "schema_hash") + if err != nil { + return err + } + + schemHashDB, err := sq.HashSqliteDatabase(tctx, tx) + if err != nil { + return err + } + + if schemHashDB != langext.Coalesce(schemaHashMeta, "") || langext.Coalesce(schemaHashMeta, "") != schema.PrimarySchema[currschema].Hash { + log.Debug().Str("schemHashDB", schemHashDB).Msg("Schema (primary db)") + log.Debug().Str("schemaHashMeta", langext.Coalesce(schemaHashMeta, "")).Msg("Schema (primary db)") + log.Debug().Str("schemaHashAsset", schema.PrimarySchema[currschema].Hash).Msg("Schema (primary db)") + return errors.New("database schema does not match (primary db)") + } else { + log.Debug().Str("schemHash", schemHashDB).Msg("Verified Schema consistency (primary db)") + } } if currschema != schema.PrimarySchemaVersion { diff --git a/scnserver/db/impl/primary/deliveries.go b/scnserver/db/impl/primary/deliveries.go index ae3d5cd..1673f98 100644 --- a/scnserver/db/impl/primary/deliveries.go +++ b/scnserver/db/impl/primary/deliveries.go @@ -5,7 +5,6 @@ import ( "blackforestbytes.com/simplecloudnotifier/db" "blackforestbytes.com/simplecloudnotifier/models" "gogs.mikescher.com/BlackForestBytes/goext/langext" - "gogs.mikescher.com/BlackForestBytes/goext/rfctime" "gogs.mikescher.com/BlackForestBytes/goext/sq" "time" ) @@ -28,7 +27,7 @@ func (db *Database) CreateRetryDelivery(ctx db.TxContext, client models.Client, TimestampFinalized: nil, Status: models.DeliveryStatusRetry, RetryCount: 0, - NextDelivery: rfctime.NewRFC3339NanoPtr(&next), + NextDelivery: models.NewSCNTimePtr(&next), FCMMessageID: nil, } @@ -76,7 +75,7 @@ func (db *Database) ListRetrieableDeliveries(ctx db.TxContext, pageSize int) ([] } return sq.QueryAll[models.Delivery](ctx, tx, "SELECT * FROM deliveries WHERE status = 'RETRY' AND next_delivery < :next ORDER BY next_delivery ASC LIMIT :lim", sq.PP{ - "next": time.Now().Format(time.RFC3339Nano), + "next": time2DB(time.Now()), "lim": pageSize, }, sq.SModeExtended, sq.Safe) } @@ -126,7 +125,7 @@ func (db *Database) SetDeliveryRetry(ctx db.TxContext, delivery models.Delivery) } _, err = tx.Exec(ctx, "UPDATE deliveries SET status = 'RETRY', next_delivery = :next, retry_count = :rc WHERE delivery_id = :did", sq.PP{ - "next": scn.NextDeliveryTimestamp(time.Now()).Format(time.RFC3339Nano), + "next": time2DB(scn.NextDeliveryTimestamp(time.Now())), "rc": delivery.RetryCount + 1, "did": delivery.DeliveryID, }) diff --git a/scnserver/db/schema/assets.go b/scnserver/db/schema/assets.go index 77c2f59..7c23909 100644 --- a/scnserver/db/schema/assets.go +++ b/scnserver/db/schema/assets.go @@ -22,12 +22,18 @@ var primarySchema4 string //go:embed primary_5.ddl var primarySchema5 string +//go:embed primary_6.ddl +var primarySchema6 string + //go:embed primary_migration_3_4.ddl var PrimaryMigration_3_4 string //go:embed primary_migration_4_5.ddl var PrimaryMigration_4_5 string +//go:embed primary_migration_5_6.ddl +var PrimaryMigration_5_6 string + //go:embed requests_1.ddl var requestsSchema1 string @@ -41,9 +47,10 @@ var PrimarySchema = map[int]Def{ 3: {primarySchema3, "65c2125ad0e12d02490cf2275f0067ef3c62a8522edf9a35ee8aa3f3c09b12e8"}, 4: {primarySchema4, "cb022156ab0e7aea39dd0c985428c43cae7d60e41ca8e9e5a84c774b3019d2ca"}, 5: {primarySchema5, "9d6217ba4a3503cfe090f72569367f95a413bb14e9effe49ffeabbf255bce8dd"}, + 6: {primarySchema6, "8e83d20bcd008082713f248ae8cd558335a37a37ce90bd8c86e782da640ee160"}, } -var PrimarySchemaVersion = 5 +var PrimarySchemaVersion = 6 var RequestsSchema = map[int]Def{ 0: {"", ""}, diff --git a/scnserver/db/schema/primary_6.ddl b/scnserver/db/schema/primary_6.ddl new file mode 100644 index 0000000..7c23663 --- /dev/null +++ b/scnserver/db/schema/primary_6.ddl @@ -0,0 +1,234 @@ +CREATE TABLE users +( + user_id TEXT NOT NULL, + + username TEXT NULL DEFAULT NULL, + + timestamp_created INTEGER NOT NULL, + timestamp_lastread INTEGER NULL DEFAULT NULL, + timestamp_lastsent INTEGER NULL DEFAULT NULL, + + messages_sent INTEGER NOT NULL DEFAULT '0', + + quota_used INTEGER NOT NULL DEFAULT '0', + quota_used_day TEXT NULL DEFAULT NULL, + + is_pro INTEGER CHECK(is_pro IN (0, 1)) NOT NULL DEFAULT 0, + pro_token TEXT NULL DEFAULT NULL, + + PRIMARY KEY (user_id) +) STRICT; +CREATE UNIQUE INDEX "idx_users_protoken" ON users (pro_token) WHERE pro_token IS NOT NULL; + + +CREATE TABLE keytokens +( + keytoken_id TEXT NOT NULL, + + timestamp_created INTEGER NOT NULL, + timestamp_lastused INTEGER NULL DEFAULT NULL, + + name TEXT NOT NULL, + + owner_user_id TEXT NOT NULL, + + all_channels INTEGER CHECK(all_channels IN (0, 1)) NOT NULL, + channels TEXT NOT NULL, + token TEXT NOT NULL, + permissions TEXT NOT NULL, + + messages_sent INTEGER NOT NULL DEFAULT '0', + + PRIMARY KEY (keytoken_id) +) STRICT; +CREATE UNIQUE INDEX "idx_keytokens_token" ON keytokens (token); + + +CREATE TABLE clients +( + client_id TEXT NOT NULL, + + user_id TEXT NOT NULL, + type TEXT CHECK(type IN ('ANDROID','IOS','LINUX','MACOS','WINDOWS')) NOT NULL, + fcm_token TEXT NOT NULL, + name TEXT NULL, + + timestamp_created INTEGER NOT NULL, + + agent_model TEXT NOT NULL, + agent_version TEXT NOT NULL, + + PRIMARY KEY (client_id) +) STRICT; +CREATE INDEX "idx_clients_userid" ON clients (user_id); +CREATE UNIQUE INDEX "idx_clients_fcmtoken" ON clients (fcm_token); + + +CREATE TABLE channels +( + channel_id TEXT NOT NULL, + + owner_user_id TEXT NOT NULL, + + internal_name TEXT NOT NULL, + display_name TEXT NOT NULL, + description_name TEXT NULL, + + subscribe_key TEXT NOT NULL, + + timestamp_created INTEGER NOT NULL, + timestamp_lastsent INTEGER NULL DEFAULT NULL, + + messages_sent INTEGER NOT NULL DEFAULT '0', + + PRIMARY KEY (channel_id) +) STRICT; +CREATE UNIQUE INDEX "idx_channels_identity" ON channels (owner_user_id, internal_name); + +CREATE TABLE subscriptions +( + subscription_id TEXT NOT NULL, + + subscriber_user_id TEXT NOT NULL, + channel_owner_user_id TEXT NOT NULL, + channel_internal_name TEXT NOT NULL, + channel_id TEXT NOT NULL, + + timestamp_created INTEGER NOT NULL, + + confirmed INTEGER CHECK(confirmed IN (0, 1)) NOT NULL, + + PRIMARY KEY (subscription_id) +) STRICT; +CREATE UNIQUE INDEX "idx_subscriptions_ref" ON subscriptions (subscriber_user_id, channel_owner_user_id, channel_internal_name); +CREATE INDEX "idx_subscriptions_chan" ON subscriptions (channel_id); +CREATE INDEX "idx_subscriptions_subuser" ON subscriptions (subscriber_user_id); +CREATE INDEX "idx_subscriptions_ownuser" ON subscriptions (channel_owner_user_id); +CREATE INDEX "idx_subscriptions_tsc" ON subscriptions (timestamp_created); +CREATE INDEX "idx_subscriptions_conf" ON subscriptions (confirmed); + + +CREATE TABLE messages +( + message_id TEXT NOT NULL, + sender_user_id TEXT NOT NULL, + channel_internal_name TEXT NOT NULL, + channel_id TEXT NOT NULL, + sender_ip TEXT NOT NULL, + sender_name TEXT NULL, + + timestamp_real INTEGER NOT NULL, + timestamp_client INTEGER NULL, + + title TEXT NOT NULL, + content TEXT NULL, + priority INTEGER CHECK(priority IN (0, 1, 2)) NOT NULL, + usr_message_id TEXT NULL, + + used_key_id TEXT NOT NULL, + + deleted INTEGER CHECK(deleted IN (0, 1)) NOT NULL DEFAULT '0', + + PRIMARY KEY (message_id) +) STRICT; +CREATE INDEX "idx_messages_channel" ON messages (channel_internal_name COLLATE BINARY); +CREATE INDEX "idx_messages_channel_nc" ON messages (channel_internal_name COLLATE NOCASE); +CREATE UNIQUE INDEX "idx_messages_idempotency" ON messages (sender_user_id, usr_message_id COLLATE BINARY); +CREATE INDEX "idx_messages_senderip" ON messages (sender_ip COLLATE BINARY); +CREATE INDEX "idx_messages_sendername" ON messages (sender_name COLLATE BINARY); +CREATE INDEX "idx_messages_sendername_nc" ON messages (sender_name COLLATE NOCASE); +CREATE INDEX "idx_messages_title" ON messages (title COLLATE BINARY); +CREATE INDEX "idx_messages_title_nc" ON messages (title COLLATE NOCASE); +CREATE INDEX "idx_messages_usedkey" ON messages (sender_user_id, used_key_id); +CREATE INDEX "idx_messages_deleted" ON messages (deleted); + + +CREATE VIRTUAL TABLE messages_fts USING fts5 +( + channel_internal_name, + sender_name, + title, + content, + + tokenize = unicode61, + content = 'messages', + content_rowid = 'rowid' +); + +CREATE TRIGGER fts_insert AFTER INSERT ON messages BEGIN + INSERT INTO messages_fts (rowid, channel_internal_name, sender_name, title, content) VALUES (new.rowid, new.channel_internal_name, new.sender_name, new.title, new.content); +END; + +CREATE TRIGGER fts_update AFTER UPDATE ON messages BEGIN + INSERT INTO messages_fts (messages_fts, rowid, channel_internal_name, sender_name, title, content) VALUES ('delete', old.rowid, old.channel_internal_name, old.sender_name, old.title, old.content); + INSERT INTO messages_fts ( rowid, channel_internal_name, sender_name, title, content) VALUES ( new.rowid, new.channel_internal_name, new.sender_name, new.title, new.content); +END; + +CREATE TRIGGER fts_delete AFTER DELETE ON messages BEGIN + INSERT INTO messages_fts (messages_fts, rowid, channel_internal_name, sender_name, title, content) VALUES ('delete', old.rowid, old.channel_internal_name, old.sender_name, old.title, old.content); +END; + + +CREATE TABLE deliveries +( + delivery_id TEXT NOT NULL, + + message_id TEXT NOT NULL, + receiver_user_id TEXT NOT NULL, + receiver_client_id TEXT NOT NULL, + + timestamp_created INTEGER NOT NULL, + timestamp_finalized INTEGER NULL, + + + status TEXT CHECK(status IN ('RETRY','SUCCESS','FAILED')) NOT NULL, + retry_count INTEGER NOT NULL DEFAULT 0, + next_delivery INTEGER NULL DEFAULT NULL, + + fcm_message_id TEXT NULL, + + PRIMARY KEY (delivery_id) +) STRICT; +CREATE INDEX "idx_deliveries_receiver" ON deliveries (message_id, receiver_client_id); + + +CREATE TABLE compat_ids +( + old INTEGER NOT NULL, + new TEXT NOT NULL, + type TEXT NOT NULL +) STRICT; +CREATE UNIQUE INDEX "idx_compatids_new" ON compat_ids (new); +CREATE UNIQUE INDEX "idx_compatids_old" ON compat_ids (old, type); + + +CREATE TABLE compat_acks +( + user_id TEXT NOT NULL, + message_id TEXT NOT NULL +) STRICT; +CREATE INDEX "idx_compatacks_userid" ON compat_acks (user_id); +CREATE UNIQUE INDEX "idx_compatacks_messageid" ON compat_acks (message_id); +CREATE UNIQUE INDEX "idx_compatacks_userid_messageid" ON compat_acks (user_id, message_id); + + +CREATE TABLE compat_clients +( + client_id TEXT NOT NULL +) STRICT; +CREATE UNIQUE INDEX "idx_compatclient_clientid" ON compat_clients (client_id); + + +CREATE TABLE `meta` +( + meta_key TEXT NOT NULL, + value_int INTEGER NULL, + value_txt TEXT NULL, + value_real REAL NULL, + value_blob BLOB NULL, + + PRIMARY KEY (meta_key) +) STRICT; + + +INSERT INTO meta (meta_key, value_int) VALUES ('schema', 3) \ No newline at end of file diff --git a/scnserver/db/schema/primary_migration_5_6.ddl b/scnserver/db/schema/primary_migration_5_6.ddl new file mode 100644 index 0000000..ac626d5 --- /dev/null +++ b/scnserver/db/schema/primary_migration_5_6.ddl @@ -0,0 +1,49 @@ + +DROP INDEX "idx_deliveries_receiver"; + +CREATE TABLE deliveries_new +( + delivery_id TEXT NOT NULL, + + message_id TEXT NOT NULL, + receiver_user_id TEXT NOT NULL, + receiver_client_id TEXT NOT NULL, + + timestamp_created INTEGER NOT NULL, + timestamp_finalized INTEGER NULL, + + + status TEXT CHECK(status IN ('RETRY','SUCCESS','FAILED')) NOT NULL, + retry_count INTEGER NOT NULL DEFAULT 0, + next_delivery INTEGER NULL DEFAULT NULL, + + fcm_message_id TEXT NULL, + + PRIMARY KEY (delivery_id) +) STRICT; + +UPDATE deliveries SET next_delivery = NULL; + +INSERT INTO deliveries_new +SELECT + delivery_id, + message_id, + receiver_user_id, + receiver_client_id, + timestamp_created, + timestamp_finalized, + status, + retry_count, + next_delivery, + fcm_message_id +FROM deliveries; + + +DROP TABLE deliveries; +ALTER TABLE deliveries_new RENAME TO deliveries; + + +CREATE INDEX "idx_deliveries_receiver" ON deliveries (message_id, receiver_client_id); + + + diff --git a/scnserver/models/delivery.go b/scnserver/models/delivery.go index e19cf67..ce51910 100644 --- a/scnserver/models/delivery.go +++ b/scnserver/models/delivery.go @@ -1,7 +1,5 @@ package models -import "gogs.mikescher.com/BlackForestBytes/goext/rfctime" - type DeliveryStatus string //@enum:type const ( @@ -11,16 +9,16 @@ const ( ) type Delivery struct { - DeliveryID DeliveryID `db:"delivery_id" json:"delivery_id"` - MessageID MessageID `db:"message_id" json:"message_id"` - ReceiverUserID UserID `db:"receiver_user_id" json:"receiver_user_id"` - ReceiverClientID ClientID `db:"receiver_client_id" json:"receiver_client_id"` - TimestampCreated SCNTime `db:"timestamp_created" json:"timestamp_created"` - TimestampFinalized *SCNTime `db:"timestamp_finalized" json:"timestamp_finalized"` - Status DeliveryStatus `db:"status" json:"status"` - RetryCount int `db:"retry_count" json:"retry_count"` - NextDelivery *rfctime.RFC3339NanoTime `db:"next_delivery" json:"next_delivery"` - FCMMessageID *string `db:"fcm_message_id" json:"fcm_message_id"` + DeliveryID DeliveryID `db:"delivery_id" json:"delivery_id"` + MessageID MessageID `db:"message_id" json:"message_id"` + ReceiverUserID UserID `db:"receiver_user_id" json:"receiver_user_id"` + ReceiverClientID ClientID `db:"receiver_client_id" json:"receiver_client_id"` + TimestampCreated SCNTime `db:"timestamp_created" json:"timestamp_created"` + TimestampFinalized *SCNTime `db:"timestamp_finalized" json:"timestamp_finalized"` + Status DeliveryStatus `db:"status" json:"status"` + RetryCount int `db:"retry_count" json:"retry_count"` + NextDelivery *SCNTime `db:"next_delivery" json:"next_delivery"` + FCMMessageID *string `db:"fcm_message_id" json:"fcm_message_id"` } func (d Delivery) MaxRetryCount() int {