From 62c9a4e734dce127d56c0777ab722b96131a96e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mike=20Schw=C3=B6rer?= Date: Wed, 8 Nov 2023 18:30:30 +0100 Subject: [PATCH] v0.0.301 pagination (page+limit) support in wmo --- .idea/golinter.xml | 6 +++ README.md | 1 + cursortoken/filter.go | 4 ++ go.mod | 6 +-- go.sum | 6 +++ goextVersion.go | 4 +- pagination/filter.go | 11 ++++++ pagination/pagination.go | 16 ++++++++ wmo/queryList.go | 8 ++-- wmo/queryPaginate.go | 84 ++++++++++++++++++++++++++++++++++++++++ 10 files changed, 137 insertions(+), 9 deletions(-) create mode 100644 .idea/golinter.xml create mode 100644 pagination/filter.go create mode 100644 pagination/pagination.go create mode 100644 wmo/queryPaginate.go diff --git a/.idea/golinter.xml b/.idea/golinter.xml new file mode 100644 index 0000000..084cb03 --- /dev/null +++ b/.idea/golinter.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/README.md b/README.md index ef884ec..2971000 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ Potentially needs `export GOPRIVATE="gogs.mikescher.com"` | | | | | mongoext | Mike | Utility/Helper functions for mongodb | | cursortoken | Mike | MongoDB cursortoken implementation | +| pagination | Mike | Pagination implementation | | | | | | totpext | Mike | Implementation of TOTP (2-Factor-Auth) | | termext | Mike | Utilities for terminals (mostly color output) | diff --git a/cursortoken/filter.go b/cursortoken/filter.go index c091f6c..e899192 100644 --- a/cursortoken/filter.go +++ b/cursortoken/filter.go @@ -4,6 +4,10 @@ import ( "go.mongodb.org/mongo-driver/mongo" ) +type RawFilter interface { + FilterQuery() mongo.Pipeline +} + type Filter interface { FilterQuery() mongo.Pipeline Pagination() (string, SortDirection, string, SortDirection) diff --git a/go.mod b/go.mod index bbf05d5..fed803e 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,10 @@ require ( github.com/jmoiron/sqlx v1.3.5 github.com/rs/xid v1.5.0 github.com/rs/zerolog v1.31.0 - go.mongodb.org/mongo-driver v1.12.1 + go.mongodb.org/mongo-driver v1.13.0 golang.org/x/crypto v0.14.0 golang.org/x/sys v0.14.0 - golang.org/x/term v0.13.0 + golang.org/x/term v0.14.0 ) require ( @@ -26,7 +26,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.2 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect diff --git a/go.sum b/go.sum index fa4f4e2..29353aa 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQs github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= @@ -116,6 +118,8 @@ github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE= go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ= +go.mongodb.org/mongo-driver v1.13.0 h1:67DgFFjYOCMWdtTEmKFpV3ffWlFnh+CYZ8ZS/tXWUfY= +go.mongodb.org/mongo-driver v1.13.0/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.5.0 h1:jpGode6huXQxcskEIpOCvrU+tzo81b6+oFLUYXWtH/Y= golang.org/x/arch v0.5.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= @@ -162,6 +166,8 @@ golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= +golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/goextVersion.go b/goextVersion.go index 8e407a8..6a522e4 100644 --- a/goextVersion.go +++ b/goextVersion.go @@ -1,5 +1,5 @@ package goext -const GoextVersion = "0.0.300" +const GoextVersion = "0.0.301" -const GoextVersionTimestamp = "2023-11-04T18:55:44+0100" +const GoextVersionTimestamp = "2023-11-08T18:30:30+0100" diff --git a/pagination/filter.go b/pagination/filter.go new file mode 100644 index 0000000..abfa678 --- /dev/null +++ b/pagination/filter.go @@ -0,0 +1,11 @@ +package pagination + +import ( + "go.mongodb.org/mongo-driver/mongo" + ct "gogs.mikescher.com/BlackForestBytes/goext/cursortoken" +) + +type Filter interface { + FilterQuery() mongo.Pipeline + Pagination() (string, ct.SortDirection) +} diff --git a/pagination/pagination.go b/pagination/pagination.go new file mode 100644 index 0000000..75eb093 --- /dev/null +++ b/pagination/pagination.go @@ -0,0 +1,16 @@ +package pagination + +type Pagination struct { + Page int `json:"page"` // page (first page == 1) + Limit int `json:"limit"` // max-page-size + TotalPages int `json:"totalPages"` // total page-count + TotalItems int `json:"totalItems"` // total items-count + CurrentPageCount int `json:"currntPageCount"` // item-count in current page ( == len(data) ) +} + +func CalcPaginationTotalPages(totalItems int, limit int) int { + if totalItems == 0 { + return 0 + } + return 1 + (totalItems-1)/limit +} diff --git a/wmo/queryList.go b/wmo/queryList.go index 3a79fb3..be44f3f 100644 --- a/wmo/queryList.go +++ b/wmo/queryList.go @@ -91,11 +91,11 @@ func (c *Coll[TData]) List(ctx context.Context, filter ct.Filter, pageSize *int, return entities, nextToken, nil } -type countRes struct { - Count int64 `bson:"c"` -} +func (c *Coll[TData]) Count(ctx context.Context, filter ct.RawFilter) (int64, error) { + type countRes struct { + Count int64 `bson:"c"` + } -func (c *Coll[TData]) Count(ctx context.Context, filter ct.Filter) (int64, error) { pipeline := filter.FilterQuery() pipeline = append(pipeline, bson.D{{Key: "$count", Value: "c"}}) diff --git a/wmo/queryPaginate.go b/wmo/queryPaginate.go new file mode 100644 index 0000000..87b6fba --- /dev/null +++ b/wmo/queryPaginate.go @@ -0,0 +1,84 @@ +package wmo + +import ( + "context" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + ct "gogs.mikescher.com/BlackForestBytes/goext/cursortoken" + "gogs.mikescher.com/BlackForestBytes/goext/exerr" + "gogs.mikescher.com/BlackForestBytes/goext/langext" + pag "gogs.mikescher.com/BlackForestBytes/goext/pagination" +) + +func (c *Coll[TData]) Paginate(ctx context.Context, filter pag.Filter, page int, limit *int) ([]TData, pag.Pagination, error) { + type totalCountResult struct { + Count int `bson:"count"` + } + + if page < 0 { + page = 1 + } + + pipelineSort := mongo.Pipeline{} + pipelineFilter := mongo.Pipeline{} + pf1 := "_id" + pd1 := ct.SortASC + + if filter != nil { + pipelineFilter = filter.FilterQuery() + pf1, pd1 = filter.Pagination() + } + + if pd1 == ct.SortASC { + pipelineSort = append(pipelineSort, bson.D{{Key: "$sort", Value: bson.D{{Key: pf1, Value: +1}}}}) + } else if pd1 == ct.SortDESC { + pipelineSort = append(pipelineSort, bson.D{{Key: "$sort", Value: bson.D{{Key: pf1, Value: -1}}}}) + } + + pipelinePaginate := mongo.Pipeline{} + pipelinePaginate = append(pipelinePaginate, bson.D{{Key: "$skip", Value: page - 1}}) + if limit != nil { + pipelinePaginate = append(pipelinePaginate, bson.D{{Key: "$limit", Value: *limit}}) + } + + pipelineCount := mongo.Pipeline{} + pipelinePaginate = append(pipelinePaginate, bson.D{{Key: "$count", Value: "count"}}) + + pipelineList := langext.ArrConcat(mongo.Pipeline{}, pipelineFilter, pipelinePaginate) + pipelineTotalCount := langext.ArrConcat(mongo.Pipeline{}, pipelineFilter, pipelineCount) + + cursorList, err := c.coll.Aggregate(ctx, pipelineList) + if err != nil { + return nil, pag.Pagination{}, exerr.Wrap(err, "mongo-aggregation failed").Any("pipeline", pipelineList).Str("collection", c.Name()).Build() + } + + entities, err := c.decodeAll(ctx, cursorList) + if err != nil { + return nil, pag.Pagination{}, exerr.Wrap(err, "failed to all-decode entities").Build() + } + + cursorTotalCount, err := c.coll.Aggregate(ctx, pipelineTotalCount) + if err != nil { + return nil, pag.Pagination{}, exerr.Wrap(err, "mongo-aggregation failed").Any("pipeline", pipelineTotalCount).Str("collection", c.Name()).Build() + } + + var tcRes totalCountResult + if cursorTotalCount.Next(ctx) { + err = cursorTotalCount.Decode(&tcRes) + if err != nil { + return nil, pag.Pagination{}, exerr.Wrap(err, "failed to decode mongo-aggregation $count result").Any("pipeline", pipelineTotalCount).Str("collection", c.Name()).Build() + } + } else { + tcRes.Count = 0 // no entries in DB + } + + paginationObj := pag.Pagination{ + Page: page, + Limit: langext.Coalesce(limit, tcRes.Count), + TotalPages: pag.CalcPaginationTotalPages(tcRes.Count, langext.Coalesce(limit, tcRes.Count)), + TotalItems: tcRes.Count, + CurrentPageCount: len(entities), + } + + return entities, paginationObj, nil +}