From 9d35916280c9ecca86bce9d32f18ab2839b42349 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20Schw=C3=B6rer?= Date: Tue, 17 Sep 2024 22:25:09 +0200 Subject: [PATCH] Fix missing field in clients struct and non-partial fcmtoken index (also streamline db migrations) --- scnserver/TODO.md | 2 + scnserver/db/impl/logs/database.go | 14 +- scnserver/db/impl/primary/database.go | 22 +- scnserver/db/impl/requests/database.go | 14 +- scnserver/db/schema/assets.go | 50 ++-- scnserver/db/schema/primary_8.ddl | 238 ++++++++++++++++++ scnserver/db/schema/primary_migration_7_8.ddl | 7 + scnserver/models/client.go | 1 + scnserver/test/database_test.go | 4 +- 9 files changed, 309 insertions(+), 43 deletions(-) create mode 100644 scnserver/db/schema/primary_8.ddl create mode 100644 scnserver/db/schema/primary_migration_7_8.ddl diff --git a/scnserver/TODO.md b/scnserver/TODO.md index c21348b..e5ecaf7 100644 --- a/scnserver/TODO.md +++ b/scnserver/TODO.md @@ -12,6 +12,8 @@ - exerr.New | exerr.Wrap + - (!!!) Run tests in pipeline !! + #### UNSURE - (?) default-priority for channels diff --git a/scnserver/db/impl/logs/database.go b/scnserver/db/impl/logs/database.go index c9f1aa1..1f8de60 100644 --- a/scnserver/db/impl/logs/database.go +++ b/scnserver/db/impl/logs/database.go @@ -121,8 +121,8 @@ func (db *Database) Migrate(outerctx context.Context) error { if currschema == 0 { log.Info().Msgf("Migrate database (initialize) [%s] %d -> %d", db.name, currschema, db.schemaVersion) - schemastr := db.schema[1].SQL - schemahash := db.schema[1].Hash + schemastr := db.schema[db.schemaVersion].SQL + schemahash := db.schema[db.schemaVersion].Hash _, err = tx.Exec(tctx, schemastr, sq.PP{}) if err != nil { @@ -177,7 +177,15 @@ func (db *Database) Migrate(outerctx context.Context) error { //goland:noinspection SqlConstantCondition,SqlWithoutWhere func (db *Database) migrateSingle(tctx *simplectx.SimpleContext, tx sq.Tx, schemaFrom int, schemaTo int) error { - // ADD MIGRATIONS HERE ... + if schemaFrom == schemaTo-1 { + + migSQL := db.schema[schemaTo].MigScript + if migSQL == "" { + return exerr.New(exerr.TypeInternal, fmt.Sprintf("missing %s migration from %d to %d", db.name, schemaFrom, schemaTo)).Build() + } + + return db.migrateBySQL(tctx, tx, migSQL, schemaFrom, schemaTo, db.schema[schemaTo].Hash, nil) + } return exerr.New(exerr.TypeInternal, fmt.Sprintf("missing %s migration from %d to %d", db.name, schemaFrom, schemaTo)).Build() } diff --git a/scnserver/db/impl/primary/database.go b/scnserver/db/impl/primary/database.go index a928e72..d5669d1 100644 --- a/scnserver/db/impl/primary/database.go +++ b/scnserver/db/impl/primary/database.go @@ -121,8 +121,8 @@ func (db *Database) Migrate(outerctx context.Context) error { if currschema == 0 { log.Info().Msgf("Migrate database (initialize) [%s] %d -> %d", db.name, currschema, db.schemaVersion) - schemastr := db.schema[1].SQL - schemahash := db.schema[1].Hash + schemastr := db.schema[db.schemaVersion].SQL + schemahash := db.schema[db.schemaVersion].Hash _, err = tx.Exec(tctx, schemastr, sq.PP{}) if err != nil { @@ -177,20 +177,14 @@ func (db *Database) Migrate(outerctx context.Context) error { //goland:noinspection SqlConstantCondition,SqlWithoutWhere func (db *Database) migrateSingle(tctx *simplectx.SimpleContext, tx sq.Tx, schemaFrom int, schemaTo int) error { - if schemaFrom == 3 && schemaTo == 4 { - return db.migrateBySQL(tctx, tx, schema.PrimaryMigration_3_4, schemaFrom, schemaTo, db.schema[schemaTo].Hash, nil) - } + if schemaFrom == schemaTo-1 { - if schemaFrom == 4 && schemaTo == 5 { - return db.migrateBySQL(tctx, tx, schema.PrimaryMigration_4_5, schemaFrom, schemaTo, db.schema[schemaTo].Hash, nil) - } + migSQL := db.schema[schemaTo].MigScript + if migSQL == "" { + return exerr.New(exerr.TypeInternal, fmt.Sprintf("missing %s migration from %d to %d", db.name, schemaFrom, schemaTo)).Build() + } - if schemaFrom == 5 && schemaTo == 6 { - return db.migrateBySQL(tctx, tx, schema.PrimaryMigration_5_6, schemaFrom, schemaTo, db.schema[schemaTo].Hash, nil) - } - - if schemaFrom == 6 && schemaTo == 7 { - return db.migrateBySQL(tctx, tx, schema.PrimaryMigration_6_7, schemaFrom, schemaTo, db.schema[schemaTo].Hash, nil) + return db.migrateBySQL(tctx, tx, migSQL, schemaFrom, schemaTo, db.schema[schemaTo].Hash, nil) } return exerr.New(exerr.TypeInternal, fmt.Sprintf("missing %s migration from %d to %d", db.name, schemaFrom, schemaTo)).Build() diff --git a/scnserver/db/impl/requests/database.go b/scnserver/db/impl/requests/database.go index 98e54e7..669b914 100644 --- a/scnserver/db/impl/requests/database.go +++ b/scnserver/db/impl/requests/database.go @@ -121,8 +121,8 @@ func (db *Database) Migrate(outerctx context.Context) error { if currschema == 0 { log.Info().Msgf("Migrate database (initialize) [%s] %d -> %d", db.name, currschema, db.schemaVersion) - schemastr := db.schema[1].SQL - schemahash := db.schema[1].Hash + schemastr := db.schema[db.schemaVersion].SQL + schemahash := db.schema[db.schemaVersion].Hash _, err = tx.Exec(tctx, schemastr, sq.PP{}) if err != nil { @@ -177,7 +177,15 @@ func (db *Database) Migrate(outerctx context.Context) error { //goland:noinspection SqlConstantCondition,SqlWithoutWhere func (db *Database) migrateSingle(tctx *simplectx.SimpleContext, tx sq.Tx, schemaFrom int, schemaTo int) error { - // ADD MIGRATIONS HERE ... + if schemaFrom == schemaTo-1 { + + migSQL := db.schema[schemaTo].MigScript + if migSQL == "" { + return exerr.New(exerr.TypeInternal, fmt.Sprintf("missing %s migration from %d to %d", db.name, schemaFrom, schemaTo)).Build() + } + + return db.migrateBySQL(tctx, tx, migSQL, schemaFrom, schemaTo, db.schema[schemaTo].Hash, nil) + } return exerr.New(exerr.TypeInternal, fmt.Sprintf("missing %s migration from %d to %d", db.name, schemaFrom, schemaTo)).Build() } diff --git a/scnserver/db/schema/assets.go b/scnserver/db/schema/assets.go index 4572ad4..78b142f 100644 --- a/scnserver/db/schema/assets.go +++ b/scnserver/db/schema/assets.go @@ -3,8 +3,9 @@ package schema import _ "embed" type Def struct { - SQL string - Hash string + SQL string + Hash string + MigScript string } //go:embed primary_1.ddl @@ -28,17 +29,23 @@ var primarySchema6 string //go:embed primary_7.ddl var primarySchema7 string +//go:embed primary_8.ddl +var primarySchema8 string + //go:embed primary_migration_3_4.ddl -var PrimaryMigration_3_4 string +var primaryMigration_3_4 string //go:embed primary_migration_4_5.ddl -var PrimaryMigration_4_5 string +var primaryMigration_4_5 string //go:embed primary_migration_5_6.ddl -var PrimaryMigration_5_6 string +var primaryMigration_5_6 string //go:embed primary_migration_6_7.ddl -var PrimaryMigration_6_7 string +var primaryMigration_6_7 string + +//go:embed primary_migration_7_8.ddl +var primaryMigration_7_8 string //go:embed requests_1.ddl var requestsSchema1 string @@ -47,28 +54,29 @@ var requestsSchema1 string var logsSchema1 string var PrimarySchema = map[int]Def{ - 0: {"", ""}, - 1: {primarySchema1, "f2b2847f32681a7178e405553beea4a324034915a0c5a5dc70b3c6abbcc852f2"}, - 2: {primarySchema2, "07ed1449114416ed043084a30e0722a5f97bf172161338d2f7106a8dfd387d0a"}, - 3: {primarySchema3, "65c2125ad0e12d02490cf2275f0067ef3c62a8522edf9a35ee8aa3f3c09b12e8"}, - 4: {primarySchema4, "cb022156ab0e7aea39dd0c985428c43cae7d60e41ca8e9e5a84c774b3019d2ca"}, - 5: {primarySchema5, "9d6217ba4a3503cfe090f72569367f95a413bb14e9effe49ffeabbf255bce8dd"}, - 6: {primarySchema6, "8e83d20bcd008082713f248ae8cd558335a37a37ce90bd8c86e782da640ee160"}, - 7: {primarySchema7, "90d8dbc460afe025f9b74cda5c16bb8e58b178df275223bd2531907a8d8c36c3"}, + 0: {"", "", ""}, + 1: {primarySchema1, "f2b2847f32681a7178e405553beea4a324034915a0c5a5dc70b3c6abbcc852f2", ""}, + 2: {primarySchema2, "07ed1449114416ed043084a30e0722a5f97bf172161338d2f7106a8dfd387d0a", ""}, + 3: {primarySchema3, "65c2125ad0e12d02490cf2275f0067ef3c62a8522edf9a35ee8aa3f3c09b12e8", ""}, + 4: {primarySchema4, "cb022156ab0e7aea39dd0c985428c43cae7d60e41ca8e9e5a84c774b3019d2ca", primaryMigration_3_4}, + 5: {primarySchema5, "9d6217ba4a3503cfe090f72569367f95a413bb14e9effe49ffeabbf255bce8dd", primaryMigration_4_5}, + 6: {primarySchema6, "8e83d20bcd008082713f248ae8cd558335a37a37ce90bd8c86e782da640ee160", primaryMigration_5_6}, + 7: {primarySchema7, "90d8dbc460afe025f9b74cda5c16bb8e58b178df275223bd2531907a8d8c36c3", primaryMigration_6_7}, + 8: {primarySchema8, "746f6005c7a573b8816e5993ecd1d949fe2552b0134ba63bab8b4d5b2b5058ad", primaryMigration_7_8}, } -var PrimarySchemaVersion = 7 +var PrimarySchemaVersion = len(PrimarySchema) - 1 var RequestsSchema = map[int]Def{ - 0: {"", ""}, - 1: {requestsSchema1, "ebb0a5748b605e8215437413b738279670190ca8159b6227cfc2aa13418b41e9"}, + 0: {"", "", ""}, + 1: {requestsSchema1, "ebb0a5748b605e8215437413b738279670190ca8159b6227cfc2aa13418b41e9", ""}, } -var RequestsSchemaVersion = 1 +var RequestsSchemaVersion = len(RequestsSchema) - 1 var LogsSchema = map[int]Def{ - 0: {"", ""}, - 1: {logsSchema1, "65fba477c04095effc3a8e1bb79fe7547b8e52e983f776f156266eddc4f201d7"}, + 0: {"", "", ""}, + 1: {logsSchema1, "65fba477c04095effc3a8e1bb79fe7547b8e52e983f776f156266eddc4f201d7", ""}, } -var LogsSchemaVersion = 1 +var LogsSchemaVersion = len(LogsSchema) - 1 diff --git a/scnserver/db/schema/primary_8.ddl b/scnserver/db/schema/primary_8.ddl new file mode 100644 index 0000000..2a32a39 --- /dev/null +++ b/scnserver/db/schema/primary_8.ddl @@ -0,0 +1,238 @@ +CREATE TABLE users +( + user_id TEXT NOT NULL, + + username TEXT NULL DEFAULT NULL, + + timestamp_created INTEGER NOT NULL, + timestamp_lastread INTEGER NULL DEFAULT NULL, + timestamp_lastsent INTEGER NULL DEFAULT NULL, + + messages_sent INTEGER NOT NULL DEFAULT '0', + + quota_used INTEGER NOT NULL DEFAULT '0', + quota_used_day TEXT NULL DEFAULT NULL, + + is_pro INTEGER CHECK(is_pro IN (0, 1)) NOT NULL DEFAULT 0, + pro_token TEXT NULL DEFAULT NULL, + + PRIMARY KEY (user_id) +) STRICT; +CREATE UNIQUE INDEX "idx_users_protoken" ON users (pro_token) WHERE pro_token IS NOT NULL; + + +CREATE TABLE keytokens +( + keytoken_id TEXT NOT NULL, + + timestamp_created INTEGER NOT NULL, + timestamp_lastused INTEGER NULL DEFAULT NULL, + + name TEXT NOT NULL, + + owner_user_id TEXT NOT NULL, + + all_channels INTEGER CHECK(all_channels IN (0, 1)) NOT NULL, + channels TEXT NOT NULL, + token TEXT NOT NULL, + permissions TEXT NOT NULL, + + messages_sent INTEGER NOT NULL DEFAULT '0', + + PRIMARY KEY (keytoken_id) +) STRICT; +CREATE UNIQUE INDEX "idx_keytokens_token" ON keytokens (token); + + +CREATE TABLE clients +( + client_id TEXT NOT NULL, + + user_id TEXT NOT NULL, + type TEXT CHECK(type IN ('ANDROID','IOS','LINUX','MACOS','WINDOWS')) NOT NULL, + fcm_token TEXT NOT NULL, + name TEXT NULL, + + timestamp_created INTEGER NOT NULL, + + agent_model TEXT NOT NULL, + agent_version TEXT NOT NULL, + + + deleted INTEGER CHECK(deleted IN (0, 1)) NOT NULL DEFAULT '0', + + PRIMARY KEY (client_id) +) STRICT; +CREATE INDEX "idx_clients_userid" ON clients (user_id); +CREATE INDEX "idx_clients_deleted" ON clients (deleted); +CREATE UNIQUE INDEX "idx_clients_fcmtoken" ON clients (fcm_token) WHERE deleted=0; + + +CREATE TABLE channels +( + channel_id TEXT NOT NULL, + + owner_user_id TEXT NOT NULL, + + internal_name TEXT NOT NULL, + display_name TEXT NOT NULL, + description_name TEXT NULL, + + subscribe_key TEXT NOT NULL, + + timestamp_created INTEGER NOT NULL, + timestamp_lastsent INTEGER NULL DEFAULT NULL, + + messages_sent INTEGER NOT NULL DEFAULT '0', + + PRIMARY KEY (channel_id) +) STRICT; +CREATE UNIQUE INDEX "idx_channels_identity" ON channels (owner_user_id, internal_name); + +CREATE TABLE subscriptions +( + subscription_id TEXT NOT NULL, + + subscriber_user_id TEXT NOT NULL, + channel_owner_user_id TEXT NOT NULL, + channel_internal_name TEXT NOT NULL, + channel_id TEXT NOT NULL, + + timestamp_created INTEGER NOT NULL, + + confirmed INTEGER CHECK(confirmed IN (0, 1)) NOT NULL, + + PRIMARY KEY (subscription_id) +) STRICT; +CREATE UNIQUE INDEX "idx_subscriptions_ref" ON subscriptions (subscriber_user_id, channel_owner_user_id, channel_internal_name); +CREATE INDEX "idx_subscriptions_chan" ON subscriptions (channel_id); +CREATE INDEX "idx_subscriptions_subuser" ON subscriptions (subscriber_user_id); +CREATE INDEX "idx_subscriptions_ownuser" ON subscriptions (channel_owner_user_id); +CREATE INDEX "idx_subscriptions_tsc" ON subscriptions (timestamp_created); +CREATE INDEX "idx_subscriptions_conf" ON subscriptions (confirmed); + + +CREATE TABLE messages +( + message_id TEXT NOT NULL, + sender_user_id TEXT NOT NULL, + channel_internal_name TEXT NOT NULL, + channel_id TEXT NOT NULL, + sender_ip TEXT NOT NULL, + sender_name TEXT NULL, + + timestamp_real INTEGER NOT NULL, + timestamp_client INTEGER NULL, + + title TEXT NOT NULL, + content TEXT NULL, + priority INTEGER CHECK(priority IN (0, 1, 2)) NOT NULL, + usr_message_id TEXT NULL, + + used_key_id TEXT NOT NULL, + + deleted INTEGER CHECK(deleted IN (0, 1)) NOT NULL DEFAULT '0', + + PRIMARY KEY (message_id) +) STRICT; +CREATE INDEX "idx_messages_channel" ON messages (channel_internal_name COLLATE BINARY); +CREATE INDEX "idx_messages_channel_nc" ON messages (channel_internal_name COLLATE NOCASE); +CREATE UNIQUE INDEX "idx_messages_idempotency" ON messages (sender_user_id, usr_message_id COLLATE BINARY); +CREATE INDEX "idx_messages_senderip" ON messages (sender_ip COLLATE BINARY); +CREATE INDEX "idx_messages_sendername" ON messages (sender_name COLLATE BINARY); +CREATE INDEX "idx_messages_sendername_nc" ON messages (sender_name COLLATE NOCASE); +CREATE INDEX "idx_messages_title" ON messages (title COLLATE BINARY); +CREATE INDEX "idx_messages_title_nc" ON messages (title COLLATE NOCASE); +CREATE INDEX "idx_messages_usedkey" ON messages (sender_user_id, used_key_id); +CREATE INDEX "idx_messages_deleted" ON messages (deleted); + + +CREATE VIRTUAL TABLE messages_fts USING fts5 +( + channel_internal_name, + sender_name, + title, + content, + + tokenize = unicode61, + content = 'messages', + content_rowid = 'rowid' +); + +CREATE TRIGGER fts_insert AFTER INSERT ON messages BEGIN + INSERT INTO messages_fts (rowid, channel_internal_name, sender_name, title, content) VALUES (new.rowid, new.channel_internal_name, new.sender_name, new.title, new.content); +END; + +CREATE TRIGGER fts_update AFTER UPDATE ON messages BEGIN + INSERT INTO messages_fts (messages_fts, rowid, channel_internal_name, sender_name, title, content) VALUES ('delete', old.rowid, old.channel_internal_name, old.sender_name, old.title, old.content); + INSERT INTO messages_fts ( rowid, channel_internal_name, sender_name, title, content) VALUES ( new.rowid, new.channel_internal_name, new.sender_name, new.title, new.content); +END; + +CREATE TRIGGER fts_delete AFTER DELETE ON messages BEGIN + INSERT INTO messages_fts (messages_fts, rowid, channel_internal_name, sender_name, title, content) VALUES ('delete', old.rowid, old.channel_internal_name, old.sender_name, old.title, old.content); +END; + + +CREATE TABLE deliveries +( + delivery_id TEXT NOT NULL, + + message_id TEXT NOT NULL, + receiver_user_id TEXT NOT NULL, + receiver_client_id TEXT NOT NULL, + + timestamp_created INTEGER NOT NULL, + timestamp_finalized INTEGER NULL, + + + status TEXT CHECK(status IN ('RETRY','SUCCESS','FAILED')) NOT NULL, + retry_count INTEGER NOT NULL DEFAULT 0, + next_delivery INTEGER NULL DEFAULT NULL, + + fcm_message_id TEXT NULL, + + PRIMARY KEY (delivery_id) +) STRICT; +CREATE INDEX "idx_deliveries_receiver" ON deliveries (message_id, receiver_client_id); + + +CREATE TABLE compat_ids +( + old INTEGER NOT NULL, + new TEXT NOT NULL, + type TEXT NOT NULL +) STRICT; +CREATE UNIQUE INDEX "idx_compatids_new" ON compat_ids (new); +CREATE UNIQUE INDEX "idx_compatids_old" ON compat_ids (old, type); + + +CREATE TABLE compat_acks +( + user_id TEXT NOT NULL, + message_id TEXT NOT NULL +) STRICT; +CREATE INDEX "idx_compatacks_userid" ON compat_acks (user_id); +CREATE UNIQUE INDEX "idx_compatacks_messageid" ON compat_acks (message_id); +CREATE UNIQUE INDEX "idx_compatacks_userid_messageid" ON compat_acks (user_id, message_id); + + +CREATE TABLE compat_clients +( + client_id TEXT NOT NULL +) STRICT; +CREATE UNIQUE INDEX "idx_compatclient_clientid" ON compat_clients (client_id); + + +CREATE TABLE `meta` +( + meta_key TEXT NOT NULL, + value_int INTEGER NULL, + value_txt TEXT NULL, + value_real REAL NULL, + value_blob BLOB NULL, + + PRIMARY KEY (meta_key) +) STRICT; + + +INSERT INTO meta (meta_key, value_int) VALUES ('schema', 3) \ No newline at end of file diff --git a/scnserver/db/schema/primary_migration_7_8.ddl b/scnserver/db/schema/primary_migration_7_8.ddl new file mode 100644 index 0000000..2f6ae4f --- /dev/null +++ b/scnserver/db/schema/primary_migration_7_8.ddl @@ -0,0 +1,7 @@ + + + +DROP INDEX "idx_clients_fcmtoken"; + +CREATE UNIQUE INDEX "idx_clients_fcmtoken" ON clients (fcm_token) WHERE deleted=0; + diff --git a/scnserver/models/client.go b/scnserver/models/client.go index e97de3c..bb3849c 100644 --- a/scnserver/models/client.go +++ b/scnserver/models/client.go @@ -19,4 +19,5 @@ type Client struct { AgentModel string `db:"agent_model" json:"agent_model"` AgentVersion string `db:"agent_version" json:"agent_version"` Name *string `db:"name" json:"name"` + Deleted bool `db:"deleted" json:"-"` } diff --git a/scnserver/test/database_test.go b/scnserver/test/database_test.go index fde75d4..084314b 100644 --- a/scnserver/test/database_test.go +++ b/scnserver/test/database_test.go @@ -293,7 +293,7 @@ func TestRequestsDB_Current(t *testing.T) { } } -func TestPrimaryDB_Migrate_from_3(t *testing.T) { +func TestPrimaryDB_Migrate_from_3_to_latest(t *testing.T) { dbf1, dbf2, dbf3, conf, stop := tt.StartSimpleTestspace(t) defer stop() @@ -389,7 +389,7 @@ func TestPrimaryDB_Migrate_from_3(t *testing.T) { } } -func TestPrimaryDB_Migrate_from_4(t *testing.T) { +func TestPrimaryDB_Migrate_from_4_to_latest(t *testing.T) { dbf1, dbf2, dbf3, conf, stop := tt.StartSimpleTestspace(t) defer stop()