Fix missing field in clients struct and non-partial fcmtoken index (also streamline db migrations)

scnserver/TODO.md

@@ -12,6 +12,8 @@
 
  - exerr.New | exerr.Wrap
 
+ - (!!!) Run tests in pipeline !!
+
 #### UNSURE
 
  - (?) default-priority for channels

scnserver/db/impl/logs/database.go

@@ -121,8 +121,8 @@ func (db *Database) Migrate(outerctx context.Context) error {
 		if currschema == 0 {
 			log.Info().Msgf("Migrate database (initialize) [%s] %d -> %d", db.name, currschema, db.schemaVersion)
 
-			schemastr := db.schema[1].SQL
+			schemastr := db.schema[db.schemaVersion].SQL
-			schemahash := db.schema[1].Hash
+			schemahash := db.schema[db.schemaVersion].Hash
 
 			_, err = tx.Exec(tctx, schemastr, sq.PP{})
 			if err != nil {
@@ -177,7 +177,15 @@ func (db *Database) Migrate(outerctx context.Context) error {
 //goland:noinspection SqlConstantCondition,SqlWithoutWhere
 func (db *Database) migrateSingle(tctx *simplectx.SimpleContext, tx sq.Tx, schemaFrom int, schemaTo int) error {
 
-	// ADD MIGRATIONS HERE ...
+	if schemaFrom == schemaTo-1 {
+
+		migSQL := db.schema[schemaTo].MigScript
+		if migSQL == "" {
+			return exerr.New(exerr.TypeInternal, fmt.Sprintf("missing %s migration from %d to %d", db.name, schemaFrom, schemaTo)).Build()
+		}
+
+		return db.migrateBySQL(tctx, tx, migSQL, schemaFrom, schemaTo, db.schema[schemaTo].Hash, nil)
+	}
 
 	return exerr.New(exerr.TypeInternal, fmt.Sprintf("missing %s migration from %d to %d", db.name, schemaFrom, schemaTo)).Build()
 }

scnserver/db/impl/primary/database.go

@@ -121,8 +121,8 @@ func (db *Database) Migrate(outerctx context.Context) error {
 		if currschema == 0 {
 			log.Info().Msgf("Migrate database (initialize) [%s] %d -> %d", db.name, currschema, db.schemaVersion)
 
-			schemastr := db.schema[1].SQL
+			schemastr := db.schema[db.schemaVersion].SQL
-			schemahash := db.schema[1].Hash
+			schemahash := db.schema[db.schemaVersion].Hash
 
 			_, err = tx.Exec(tctx, schemastr, sq.PP{})
 			if err != nil {
@@ -177,20 +177,14 @@ func (db *Database) Migrate(outerctx context.Context) error {
 //goland:noinspection SqlConstantCondition,SqlWithoutWhere
 func (db *Database) migrateSingle(tctx *simplectx.SimpleContext, tx sq.Tx, schemaFrom int, schemaTo int) error {
 
-	if schemaFrom == 3 && schemaTo == 4 {
+	if schemaFrom == schemaTo-1 {
-		return db.migrateBySQL(tctx, tx, schema.PrimaryMigration_3_4, schemaFrom, schemaTo, db.schema[schemaTo].Hash, nil)
-	}
 
-	if schemaFrom == 4 && schemaTo == 5 {
+		migSQL := db.schema[schemaTo].MigScript
-		return db.migrateBySQL(tctx, tx, schema.PrimaryMigration_4_5, schemaFrom, schemaTo, db.schema[schemaTo].Hash, nil)
+		if migSQL == "" {
-	}
+			return exerr.New(exerr.TypeInternal, fmt.Sprintf("missing %s migration from %d to %d", db.name, schemaFrom, schemaTo)).Build()
+		}
 
-	if schemaFrom == 5 && schemaTo == 6 {
+		return db.migrateBySQL(tctx, tx, migSQL, schemaFrom, schemaTo, db.schema[schemaTo].Hash, nil)
-		return db.migrateBySQL(tctx, tx, schema.PrimaryMigration_5_6, schemaFrom, schemaTo, db.schema[schemaTo].Hash, nil)
-	}
-
-	if schemaFrom == 6 && schemaTo == 7 {
-		return db.migrateBySQL(tctx, tx, schema.PrimaryMigration_6_7, schemaFrom, schemaTo, db.schema[schemaTo].Hash, nil)
 	}
 
 	return exerr.New(exerr.TypeInternal, fmt.Sprintf("missing %s migration from %d to %d", db.name, schemaFrom, schemaTo)).Build()

scnserver/db/impl/requests/database.go

@@ -121,8 +121,8 @@ func (db *Database) Migrate(outerctx context.Context) error {
 		if currschema == 0 {
 			log.Info().Msgf("Migrate database (initialize) [%s] %d -> %d", db.name, currschema, db.schemaVersion)
 
-			schemastr := db.schema[1].SQL
+			schemastr := db.schema[db.schemaVersion].SQL
-			schemahash := db.schema[1].Hash
+			schemahash := db.schema[db.schemaVersion].Hash
 
 			_, err = tx.Exec(tctx, schemastr, sq.PP{})
 			if err != nil {
@@ -177,7 +177,15 @@ func (db *Database) Migrate(outerctx context.Context) error {
 //goland:noinspection SqlConstantCondition,SqlWithoutWhere
 func (db *Database) migrateSingle(tctx *simplectx.SimpleContext, tx sq.Tx, schemaFrom int, schemaTo int) error {
 
-	// ADD MIGRATIONS HERE ...
+	if schemaFrom == schemaTo-1 {
+
+		migSQL := db.schema[schemaTo].MigScript
+		if migSQL == "" {
+			return exerr.New(exerr.TypeInternal, fmt.Sprintf("missing %s migration from %d to %d", db.name, schemaFrom, schemaTo)).Build()
+		}
+
+		return db.migrateBySQL(tctx, tx, migSQL, schemaFrom, schemaTo, db.schema[schemaTo].Hash, nil)
+	}
 
 	return exerr.New(exerr.TypeInternal, fmt.Sprintf("missing %s migration from %d to %d", db.name, schemaFrom, schemaTo)).Build()
 }

scnserver/db/schema/assets.go

@@ -3,8 +3,9 @@ package schema
 import _ "embed"
 
 type Def struct {
-	SQL  string
+	SQL       string
-	Hash string
+	Hash      string
+	MigScript string
 }
 
 //go:embed primary_1.ddl
@@ -28,17 +29,23 @@ var primarySchema6 string
 //go:embed primary_7.ddl
 var primarySchema7 string
 
+//go:embed primary_8.ddl
+var primarySchema8 string
+
 //go:embed primary_migration_3_4.ddl
-var PrimaryMigration_3_4 string
+var primaryMigration_3_4 string
 
 //go:embed primary_migration_4_5.ddl
-var PrimaryMigration_4_5 string
+var primaryMigration_4_5 string
 
 //go:embed primary_migration_5_6.ddl
-var PrimaryMigration_5_6 string
+var primaryMigration_5_6 string
 
 //go:embed primary_migration_6_7.ddl
-var PrimaryMigration_6_7 string
+var primaryMigration_6_7 string
+
+//go:embed primary_migration_7_8.ddl
+var primaryMigration_7_8 string
 
 //go:embed requests_1.ddl
 var requestsSchema1 string
@@ -47,28 +54,29 @@ var requestsSchema1 string
 var logsSchema1 string
 
 var PrimarySchema = map[int]Def{
-	0: {"", ""},
+	0: {"", "", ""},
-	1: {primarySchema1, "f2b2847f32681a7178e405553beea4a324034915a0c5a5dc70b3c6abbcc852f2"},
+	1: {primarySchema1, "f2b2847f32681a7178e405553beea4a324034915a0c5a5dc70b3c6abbcc852f2", ""},
-	2: {primarySchema2, "07ed1449114416ed043084a30e0722a5f97bf172161338d2f7106a8dfd387d0a"},
+	2: {primarySchema2, "07ed1449114416ed043084a30e0722a5f97bf172161338d2f7106a8dfd387d0a", ""},
-	3: {primarySchema3, "65c2125ad0e12d02490cf2275f0067ef3c62a8522edf9a35ee8aa3f3c09b12e8"},
+	3: {primarySchema3, "65c2125ad0e12d02490cf2275f0067ef3c62a8522edf9a35ee8aa3f3c09b12e8", ""},
-	4: {primarySchema4, "cb022156ab0e7aea39dd0c985428c43cae7d60e41ca8e9e5a84c774b3019d2ca"},
+	4: {primarySchema4, "cb022156ab0e7aea39dd0c985428c43cae7d60e41ca8e9e5a84c774b3019d2ca", primaryMigration_3_4},
-	5: {primarySchema5, "9d6217ba4a3503cfe090f72569367f95a413bb14e9effe49ffeabbf255bce8dd"},
+	5: {primarySchema5, "9d6217ba4a3503cfe090f72569367f95a413bb14e9effe49ffeabbf255bce8dd", primaryMigration_4_5},
-	6: {primarySchema6, "8e83d20bcd008082713f248ae8cd558335a37a37ce90bd8c86e782da640ee160"},
+	6: {primarySchema6, "8e83d20bcd008082713f248ae8cd558335a37a37ce90bd8c86e782da640ee160", primaryMigration_5_6},
-	7: {primarySchema7, "90d8dbc460afe025f9b74cda5c16bb8e58b178df275223bd2531907a8d8c36c3"},
+	7: {primarySchema7, "90d8dbc460afe025f9b74cda5c16bb8e58b178df275223bd2531907a8d8c36c3", primaryMigration_6_7},
+	8: {primarySchema8, "746f6005c7a573b8816e5993ecd1d949fe2552b0134ba63bab8b4d5b2b5058ad", primaryMigration_7_8},
 }
 
-var PrimarySchemaVersion = 7
+var PrimarySchemaVersion = len(PrimarySchema) - 1
 
 var RequestsSchema = map[int]Def{
-	0: {"", ""},
+	0: {"", "", ""},
-	1: {requestsSchema1, "ebb0a5748b605e8215437413b738279670190ca8159b6227cfc2aa13418b41e9"},
+	1: {requestsSchema1, "ebb0a5748b605e8215437413b738279670190ca8159b6227cfc2aa13418b41e9", ""},
 }
 
-var RequestsSchemaVersion = 1
+var RequestsSchemaVersion = len(RequestsSchema) - 1
 
 var LogsSchema = map[int]Def{
-	0: {"", ""},
+	0: {"", "", ""},
-	1: {logsSchema1, "65fba477c04095effc3a8e1bb79fe7547b8e52e983f776f156266eddc4f201d7"},
+	1: {logsSchema1, "65fba477c04095effc3a8e1bb79fe7547b8e52e983f776f156266eddc4f201d7", ""},
 }
 
-var LogsSchemaVersion = 1
+var LogsSchemaVersion = len(LogsSchema) - 1

scnserver/db/schema/primary_8.ddl

@@ -0,0 +1,238 @@
+CREATE TABLE users
+(
+    user_id            TEXT                                NOT NULL,
+
+    username           TEXT                                    NULL  DEFAULT NULL,
+
+    timestamp_created  INTEGER                             NOT NULL,
+    timestamp_lastread INTEGER                                 NULL   DEFAULT NULL,
+    timestamp_lastsent INTEGER                                 NULL   DEFAULT NULL,
+
+    messages_sent      INTEGER                             NOT NULL   DEFAULT '0',
+
+    quota_used         INTEGER                             NOT NULL   DEFAULT '0',
+    quota_used_day     TEXT                                    NULL   DEFAULT NULL,
+
+    is_pro             INTEGER   CHECK(is_pro IN (0, 1))   NOT NULL   DEFAULT 0,
+    pro_token          TEXT                                    NULL   DEFAULT NULL,
+
+    PRIMARY KEY (user_id)
+) STRICT;
+CREATE  UNIQUE INDEX "idx_users_protoken" ON users (pro_token) WHERE pro_token IS NOT NULL;
+
+
+CREATE TABLE keytokens
+(
+    keytoken_id        TEXT                                      NOT NULL,
+
+    timestamp_created  INTEGER                                   NOT NULL,
+    timestamp_lastused INTEGER                                       NULL   DEFAULT NULL,
+
+    name               TEXT                                      NOT NULL,
+
+    owner_user_id      TEXT                                      NOT NULL,
+
+    all_channels       INTEGER   CHECK(all_channels IN (0, 1))   NOT NULL,
+    channels           TEXT                                      NOT NULL,
+    token              TEXT                                      NOT NULL,
+    permissions        TEXT                                      NOT NULL,
+
+    messages_sent      INTEGER                                   NOT NULL   DEFAULT '0',
+
+    PRIMARY KEY (keytoken_id)
+) STRICT;
+CREATE  UNIQUE INDEX "idx_keytokens_token" ON keytokens (token);
+
+
+CREATE TABLE clients
+(
+    client_id          TEXT                                                                    NOT NULL,
+
+    user_id            TEXT                                                                    NOT NULL,
+    type               TEXT       CHECK(type IN ('ANDROID','IOS','LINUX','MACOS','WINDOWS'))   NOT NULL,
+    fcm_token          TEXT                                                                    NOT NULL,
+    name               TEXT                                                                        NULL,
+
+    timestamp_created  INTEGER                                                                 NOT NULL,
+
+    agent_model        TEXT                                                                    NOT NULL,
+    agent_version      TEXT                                                                    NOT NULL,
+
+
+    deleted               INTEGER  CHECK(deleted IN (0, 1))        NOT NULL    DEFAULT '0',
+
+    PRIMARY KEY (client_id)
+) STRICT;
+CREATE        INDEX "idx_clients_userid"   ON clients (user_id);
+CREATE        INDEX "idx_clients_deleted"  ON clients (deleted);
+CREATE UNIQUE INDEX "idx_clients_fcmtoken" ON clients (fcm_token) WHERE deleted=0;
+
+
+CREATE TABLE channels
+(
+    channel_id         TEXT        NOT NULL,
+
+    owner_user_id      TEXT        NOT NULL,
+
+    internal_name      TEXT        NOT NULL,
+    display_name       TEXT        NOT NULL,
+    description_name   TEXT            NULL,
+
+    subscribe_key      TEXT        NOT NULL,
+
+    timestamp_created  INTEGER     NOT NULL,
+    timestamp_lastsent INTEGER         NULL   DEFAULT NULL,
+
+    messages_sent      INTEGER     NOT NULL   DEFAULT '0',
+
+    PRIMARY KEY (channel_id)
+) STRICT;
+CREATE UNIQUE INDEX "idx_channels_identity" ON channels (owner_user_id, internal_name);
+
+CREATE TABLE subscriptions
+(
+    subscription_id        TEXT                                   NOT NULL,
+
+    subscriber_user_id     TEXT                                   NOT NULL,
+    channel_owner_user_id  TEXT                                   NOT NULL,
+    channel_internal_name  TEXT                                   NOT NULL,
+    channel_id             TEXT                                   NOT NULL,
+
+    timestamp_created      INTEGER                                NOT NULL,
+
+    confirmed              INTEGER   CHECK(confirmed IN (0, 1))   NOT NULL,
+
+    PRIMARY KEY (subscription_id)
+) STRICT;
+CREATE UNIQUE INDEX "idx_subscriptions_ref"     ON subscriptions (subscriber_user_id, channel_owner_user_id, channel_internal_name);
+CREATE        INDEX "idx_subscriptions_chan"    ON subscriptions (channel_id);
+CREATE        INDEX "idx_subscriptions_subuser" ON subscriptions (subscriber_user_id);
+CREATE        INDEX "idx_subscriptions_ownuser" ON subscriptions (channel_owner_user_id);
+CREATE        INDEX "idx_subscriptions_tsc"     ON subscriptions (timestamp_created);
+CREATE        INDEX "idx_subscriptions_conf"    ON subscriptions (confirmed);
+
+
+CREATE TABLE messages
+(
+    message_id            TEXT                                     NOT NULL,
+    sender_user_id        TEXT                                     NOT NULL,
+    channel_internal_name TEXT                                     NOT NULL,
+    channel_id            TEXT                                     NOT NULL,
+    sender_ip             TEXT                                     NOT NULL,
+    sender_name           TEXT                                         NULL,
+
+    timestamp_real        INTEGER                                  NOT NULL,
+    timestamp_client      INTEGER                                      NULL,
+
+    title                 TEXT                                     NOT NULL,
+    content               TEXT                                         NULL,
+    priority              INTEGER  CHECK(priority IN (0, 1, 2))    NOT NULL,
+    usr_message_id        TEXT                                         NULL,
+
+    used_key_id           TEXT                                     NOT NULL,
+
+    deleted               INTEGER  CHECK(deleted IN (0, 1))        NOT NULL    DEFAULT '0',
+
+    PRIMARY KEY (message_id)
+) STRICT;
+CREATE        INDEX "idx_messages_channel"          ON messages (channel_internal_name COLLATE BINARY);
+CREATE        INDEX "idx_messages_channel_nc"       ON messages (channel_internal_name COLLATE NOCASE);
+CREATE UNIQUE INDEX "idx_messages_idempotency"      ON messages (sender_user_id, usr_message_id COLLATE BINARY);
+CREATE        INDEX "idx_messages_senderip"         ON messages (sender_ip COLLATE BINARY);
+CREATE        INDEX "idx_messages_sendername"       ON messages (sender_name COLLATE BINARY);
+CREATE        INDEX "idx_messages_sendername_nc"    ON messages (sender_name COLLATE NOCASE);
+CREATE        INDEX "idx_messages_title"            ON messages (title COLLATE BINARY);
+CREATE        INDEX "idx_messages_title_nc"         ON messages (title COLLATE NOCASE);
+CREATE        INDEX "idx_messages_usedkey"          ON messages (sender_user_id, used_key_id);
+CREATE        INDEX "idx_messages_deleted"          ON messages (deleted);
+
+
+CREATE VIRTUAL TABLE messages_fts USING fts5
+(
+    channel_internal_name,
+    sender_name,
+    title,
+    content,
+
+    tokenize = unicode61,
+    content = 'messages',
+    content_rowid = 'rowid'
+);
+
+CREATE TRIGGER fts_insert AFTER INSERT ON messages BEGIN
+    INSERT INTO messages_fts (rowid, channel_internal_name, sender_name, title, content) VALUES (new.rowid, new.channel_internal_name, new.sender_name, new.title, new.content);
+END;
+
+CREATE TRIGGER fts_update AFTER UPDATE ON messages BEGIN
+    INSERT INTO messages_fts (messages_fts, rowid, channel_internal_name, sender_name, title, content) VALUES ('delete', old.rowid, old.channel_internal_name, old.sender_name, old.title, old.content);
+    INSERT INTO messages_fts (              rowid, channel_internal_name, sender_name, title, content) VALUES (          new.rowid, new.channel_internal_name, new.sender_name, new.title, new.content);
+END;
+
+CREATE TRIGGER fts_delete AFTER DELETE ON messages BEGIN
+    INSERT INTO messages_fts (messages_fts, rowid, channel_internal_name, sender_name, title, content) VALUES ('delete', old.rowid, old.channel_internal_name, old.sender_name, old.title, old.content);
+END;
+
+
+CREATE TABLE deliveries
+(
+    delivery_id         TEXT                                                     NOT NULL,
+
+    message_id          TEXT                                                     NOT NULL,
+    receiver_user_id    TEXT                                                     NOT NULL,
+    receiver_client_id  TEXT                                                     NOT NULL,
+
+    timestamp_created   INTEGER                                                  NOT NULL,
+    timestamp_finalized INTEGER                                                      NULL,
+
+
+    status              TEXT     CHECK(status IN ('RETRY','SUCCESS','FAILED'))   NOT NULL,
+    retry_count         INTEGER                                                  NOT NULL   DEFAULT 0,
+    next_delivery       INTEGER                                                      NULL   DEFAULT NULL,
+
+    fcm_message_id      TEXT                                                         NULL,
+
+    PRIMARY KEY (delivery_id)
+) STRICT;
+CREATE INDEX "idx_deliveries_receiver" ON deliveries (message_id, receiver_client_id);
+
+
+CREATE TABLE compat_ids
+(
+    old         INTEGER NOT NULL,
+    new         TEXT    NOT NULL,
+    type        TEXT    NOT NULL
+) STRICT;
+CREATE UNIQUE INDEX "idx_compatids_new" ON compat_ids (new);
+CREATE UNIQUE INDEX "idx_compatids_old" ON compat_ids (old, type);
+
+
+CREATE TABLE compat_acks
+(
+    user_id         TEXT NOT NULL,
+    message_id      TEXT NOT NULL
+) STRICT;
+CREATE        INDEX "idx_compatacks_userid"           ON compat_acks (user_id);
+CREATE UNIQUE INDEX "idx_compatacks_messageid"        ON compat_acks (message_id);
+CREATE UNIQUE INDEX "idx_compatacks_userid_messageid" ON compat_acks (user_id, message_id);
+
+
+CREATE TABLE compat_clients
+(
+    client_id         TEXT    NOT NULL
+) STRICT;
+CREATE UNIQUE INDEX "idx_compatclient_clientid" ON compat_clients (client_id);
+
+
+CREATE TABLE `meta`
+(
+    meta_key     TEXT       NOT NULL,
+    value_int    INTEGER        NULL,
+    value_txt    TEXT           NULL,
+    value_real   REAL           NULL,
+    value_blob   BLOB           NULL,
+
+    PRIMARY KEY (meta_key)
+) STRICT;
+
+
+INSERT INTO meta (meta_key, value_int) VALUES ('schema', 3)

scnserver/db/schema/primary_migration_7_8.ddl

@@ -0,0 +1,7 @@
+
+
+
+DROP INDEX "idx_clients_fcmtoken";
+
+CREATE UNIQUE INDEX "idx_clients_fcmtoken" ON clients (fcm_token) WHERE deleted=0;
+

scnserver/models/client.go

@@ -19,4 +19,5 @@ type Client struct {
 	AgentModel       string     `db:"agent_model"       json:"agent_model"`
 	AgentVersion     string     `db:"agent_version"     json:"agent_version"`
 	Name             *string    `db:"name"              json:"name"`
+	Deleted          bool       `db:"deleted"           json:"-"`
 }

scnserver/test/database_test.go

@@ -293,7 +293,7 @@ func TestRequestsDB_Current(t *testing.T) {
 	}
 }
 
-func TestPrimaryDB_Migrate_from_3(t *testing.T) {
+func TestPrimaryDB_Migrate_from_3_to_latest(t *testing.T) {
 	dbf1, dbf2, dbf3, conf, stop := tt.StartSimpleTestspace(t)
 	defer stop()
 
@@ -389,7 +389,7 @@ func TestPrimaryDB_Migrate_from_3(t *testing.T) {
 	}
 }
 
-func TestPrimaryDB_Migrate_from_4(t *testing.T) {
+func TestPrimaryDB_Migrate_from_4_to_latest(t *testing.T) {
 	dbf1, dbf2, dbf3, conf, stop := tt.StartSimpleTestspace(t)
 	defer stop()