From 764ce79a7156bc0afe9a548d0363e8732b9203cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20Schw=C3=B6rer?= Date: Tue, 23 Apr 2024 16:12:17 +0200 Subject: [PATCH] v0.0.437 properly handle $group in wmo extraModPipeline --- go.mod | 2 +- go.sum | 7 +++++++ goextVersion.go | 4 ++-- wmo/collection.go | 14 ++++++++++++++ wmo/queryFind.go | 8 ++++++++ wmo/queryList.go | 18 ++++++++++++------ 6 files changed, 44 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 0d26bf5..8bc0919 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22 require ( github.com/gin-gonic/gin v1.9.1 github.com/glebarez/go-sqlite v1.22.0 // only needed for tests -.- - github.com/jmoiron/sqlx v1.3.5 + github.com/jmoiron/sqlx v1.4.0 github.com/rs/xid v1.5.0 github.com/rs/zerolog v1.32.0 go.mongodb.org/mongo-driver v1.15.0 diff --git a/go.sum b/go.sum index df5f272..e474413 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= github.com/bytedance/sonic v1.10.0-rc/go.mod h1:ElCzW+ufi8qKqNW0FY314xriJhyJhuoJ3gFZdAHF7NM= github.com/bytedance/sonic v1.10.2 h1:GQebETVBxYB7JGWJtLBi07OVzWwt+8dWA00gEVW2ZFE= @@ -63,6 +64,8 @@ github.com/go-playground/validator/v10 v10.19.0 h1:ol+5Fu+cSq9JD7SoSqe04GMI92cbn github.com/go-playground/validator/v10 v10.19.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -80,6 +83,8 @@ github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= +github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= +github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= @@ -103,6 +108,7 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -112,6 +118,7 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/goextVersion.go b/goextVersion.go index 3328286..52fcd17 100644 --- a/goextVersion.go +++ b/goextVersion.go @@ -1,5 +1,5 @@ package goext -const GoextVersion = "0.0.436" +const GoextVersion = "0.0.437" -const GoextVersionTimestamp = "2024-04-18T14:09:26+0200" +const GoextVersionTimestamp = "2024-04-23T16:12:17+0200" diff --git a/wmo/collection.go b/wmo/collection.go index 121f31d..bbfdc27 100644 --- a/wmo/collection.go +++ b/wmo/collection.go @@ -125,3 +125,17 @@ func (c *Coll[TData]) createToken(fieldPrimary string, dirPrimary ct.SortDirecti Extra: ct.Extra{}, }, nil } + +func (c *Coll[TData]) needsDoubleSort(ctx context.Context) bool { + for _, ppl := range c.extraModPipeline { + for _, stage := range ppl(ctx) { + for _, bsone := range stage { + if bsone.Key == "$group" { + // a group stage in extraModPipeline results in unsorted data, which means the caller must sort again after these pipeline stages... + return true + } + } + } + } + return false +} diff --git a/wmo/queryFind.go b/wmo/queryFind.go index a6b5b30..3b71d2e 100644 --- a/wmo/queryFind.go +++ b/wmo/queryFind.go @@ -36,6 +36,14 @@ func (c *Coll[TData]) Find(ctx context.Context, filter bson.M, opts ...*options. pipeline = langext.ArrConcat(pipeline, ppl(ctx)) } + if c.needsDoubleSort(ctx) { + for _, opt := range opts { + if opt != nil && opt.Sort != nil { + pipeline = append(pipeline, bson.D{{Key: "$sort", Value: opt.Sort}}) + } + } + } + for _, opt := range opts { if opt != nil && opt.Projection != nil { pipeline = append(pipeline, bson.D{{Key: "$project", Value: opt.Projection}}) diff --git a/wmo/queryList.go b/wmo/queryList.go index 1297c8c..1d4dedd 100644 --- a/wmo/queryList.go +++ b/wmo/queryList.go @@ -35,7 +35,7 @@ func (c *Coll[TData]) List(ctx context.Context, filter ct.Filter, pageSize *int, sortDirSecondary = nil } - paginationPipeline, err := createPaginationPipeline(c, inTok, sortPrimary, sortDirPrimary, sortSecondary, sortDirSecondary, pageSize) + paginationPipeline, doubleSortPipeline, err := createPaginationPipeline(c, inTok, sortPrimary, sortDirPrimary, sortSecondary, sortDirSecondary, pageSize) if err != nil { return nil, ct.CursorToken{}, exerr. Wrap(err, "failed to create pagination"). @@ -56,6 +56,10 @@ func (c *Coll[TData]) List(ctx context.Context, filter ct.Filter, pageSize *int, pipeline = langext.ArrConcat(pipeline, ppl(ctx)) } + if c.needsDoubleSort(ctx) { + pipeline = langext.ArrConcat(pipeline, doubleSortPipeline) + } + cursor, err := c.coll.Aggregate(ctx, pipeline) if err != nil { return nil, ct.CursorToken{}, exerr.Wrap(err, "mongo-aggregation failed").Any("pipeline", pipeline).Str("collection", c.Name()).Build() @@ -136,14 +140,14 @@ func (c *Coll[TData]) ListWithCount(ctx context.Context, filter ct.Filter, pageS return data, token, count, nil } -func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CursorToken, fieldPrimary string, sortPrimary ct.SortDirection, fieldSecondary *string, sortSecondary *ct.SortDirection, pageSize *int) ([]bson.D, error) { +func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CursorToken, fieldPrimary string, sortPrimary ct.SortDirection, fieldSecondary *string, sortSecondary *ct.SortDirection, pageSize *int) ([]bson.D, []bson.D, error) { cond := bson.A{} sort := bson.D{} valuePrimary, err := coll.getTokenValueAsMongoType(token.ValuePrimary, fieldPrimary) if err != nil { - return nil, exerr.Wrap(err, "failed to get (primary) token-value as mongo-type").Build() + return nil, nil, exerr.Wrap(err, "failed to get (primary) token-value as mongo-type").Build() } if sortPrimary == ct.SortASC { @@ -160,7 +164,7 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CursorToken valueSecondary, err := coll.getTokenValueAsMongoType(token.ValueSecondary, *fieldSecondary) if err != nil { - return nil, exerr.Wrap(err, "failed to get (secondary) token-value as mongo-type").Build() + return nil, nil, exerr.Wrap(err, "failed to get (secondary) token-value as mongo-type").Build() } if *sortSecondary == ct.SortASC { @@ -203,15 +207,17 @@ func createPaginationPipeline[TData any](coll *Coll[TData], token ct.CursorToken } else { - return nil, exerr.New(exerr.TypeInternal, "unknown ct mode: "+string(token.Mode)).Any("token.Mode", token.Mode).Build() + return nil, nil, exerr.New(exerr.TypeInternal, "unknown ct mode: "+string(token.Mode)).Any("token.Mode", token.Mode).Build() } pipeline = append(pipeline, bson.D{{Key: "$sort", Value: sort}}) + pipelineSort := mongo.Pipeline{bson.D{{Key: "$sort", Value: sort}}} + if pageSize != nil { pipeline = append(pipeline, bson.D{{Key: "$limit", Value: int64(*pageSize + 1)}}) } - return pipeline, nil + return pipeline, pipelineSort, nil }