feat(page): add typed keyset pagination for Bob (#15)
Documentation / docs (push) Successful in 29s
ci / check (push) Successful in 41s
ci / postgres (push) Successful in 45s

Add transport-neutral keyset pagination through native Bob query mods.
Preserve supported cursor value types in a compact versioned format, with
bidirectional pages, explicit row keys, and a separate opt-in count.
Include SQLite tests, PostgreSQL service CI, and the cursor-format ADR.

Closes #6
This commit was merged in pull request #15.
This commit is contained in:
end committed 2026-10-27 20:21:58 -07:00
1 parent e5298cff37
commit c45e8abe08
11 files changed
+1225 -1

No files matched your search

+216
View File
@@ -0,0 +1,216 @@
package page
import (
"bytes"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"strconv"
"time"
"unicode/utf8"
"uuid"
)
const maxCursorBytes = 16384
// CursorError identifies malformed, version, sort, value, or size cursor failures via
// Kind. It intentionally omits cursor payloads from its message.
type CursorError struct{ Kind string }
// Error implements error without including cursor contents.
func (e *CursorError) Error() string { return "page: cursor " + e.Kind }
// ValueError identifies an unsupported concrete type or invalid supported value
// at Position in the row key, without exposing the value itself.
type ValueError struct {
Position int
Type string
}
// Error implements error without including key values.
func (e *ValueError) Error() string {
return fmt.Sprintf("page: invalid key value at %d (%s)", e.Position, e.Type)
}
type envelope struct {
Version int `json:"v"`
Sort string `json:"sort"`
Values [][2]string `json:"values"`
}
func encodeCursor(sortID string, values []any) (string, error) {
e := envelope{Version: 1, Sort: sortID, Values: make([][2]string, len(values))}
for i, v := range values {
pair, ok := encodeValue(v)
if !ok {
return "", &ValueError{Position: i, Type: fmt.Sprintf("%T", v)}
}
e.Values[i] = pair
}
data, err := json.Marshal(e)
if err != nil {
return "", &CursorError{Kind: "malformed"}
}
if base64.RawURLEncoding.EncodedLen(len(data)) > maxCursorBytes {
return "", &CursorError{Kind: "size"}
}
return base64.RawURLEncoding.EncodeToString(data), nil
}
func encodeValue(value any) ([2]string, bool) {
var tag, payload string
switch v := value.(type) {
case int:
tag, payload = "int", strconv.FormatInt(int64(v), 10)
case int8:
tag, payload = "int8", strconv.FormatInt(int64(v), 10)
case int16:
tag, payload = "int16", strconv.FormatInt(int64(v), 10)
case int32:
tag, payload = "int32", strconv.FormatInt(int64(v), 10)
case int64:
tag, payload = "int64", strconv.FormatInt(v, 10)
case string:
if !utf8.ValidString(v) {
return [2]string{}, false
}
tag, payload = "string", v
case time.Time:
v = v.UTC()
if v.Year() < 0 || v.Year() > 9999 {
return [2]string{}, false
}
tag, payload = "time", v.Format(time.RFC3339Nano)
case uuid.UUID:
tag, payload = "uuid", v.String()
default:
return [2]string{}, false
}
return [2]string{tag, payload}, true
}
func decodeCursor(token string, sort Sort) ([]any, error) {
failure := func(kind string) ([]any, error) { return nil, &CursorError{Kind: kind} }
if len(token) > maxCursorBytes {
return failure("size")
}
data, err := base64.RawURLEncoding.Strict().DecodeString(token)
if err != nil || !utf8.Valid(data) || base64.RawURLEncoding.EncodeToString(data) != token {
return failure("malformed")
}
// invariant: exactly three distinct envelope fields; struct decoding alone permits duplicates.
decoder := json.NewDecoder(bytes.NewReader(data))
start, err := decoder.Token()
if err != nil || start != json.Delim('{') {
return failure("malformed")
}
fields := make(map[string]json.RawMessage, 3)
for decoder.More() {
name, err := decoder.Token()
if err != nil {
return failure("malformed")
}
key, ok := name.(string)
if !ok || (key != "v" && key != "sort" && key != "values") {
return failure("malformed")
}
if _, exists := fields[key]; exists {
return failure("malformed")
}
var raw json.RawMessage
if decoder.Decode(&raw) != nil {
return failure("malformed")
}
fields[key] = raw
}
if _, err := decoder.Token(); err != nil || len(fields) != 3 {
return failure("malformed")
}
if _, err := decoder.Token(); err != io.EOF {
return failure("malformed")
}
var version int
if bytes.Equal(fields["v"], []byte("null")) || json.Unmarshal(fields["v"], &version) != nil {
return failure("malformed")
}
if version != 1 {
return failure("version")
}
var id string
if !jsonString(fields["sort"], &id) {
return failure("malformed")
}
if id != sort.ID {
return failure("sort")
}
var entries []json.RawMessage
if len(fields["values"]) == 0 || fields["values"][0] != '[' || json.Unmarshal(fields["values"], &entries) != nil || len(entries) != len(sort.Columns) {
return failure("malformed")
}
values := make([]any, len(entries))
for i, entry := range entries {
var pair []json.RawMessage
if len(entry) == 0 || entry[0] != '[' || json.Unmarshal(entry, &pair) != nil || len(pair) != 2 {
return failure("malformed")
}
var tag, payload string
if !jsonString(pair[0], &tag) || !jsonString(pair[1], &payload) {
return failure("malformed")
}
value, ok := decodeValue(tag, payload)
if !ok {
return failure("value")
}
values[i] = value
}
return values, nil
}
func jsonString(raw json.RawMessage, value *string) bool {
return len(raw) > 0 && raw[0] == '"' && json.Unmarshal(raw, value) == nil && utf8.ValidString(*value)
}
func decodeValue(tag, payload string) (any, bool) {
switch tag {
case "int", "int8", "int16", "int32", "int64":
bits := strconv.IntSize
switch tag {
case "int8":
bits = 8
case "int16":
bits = 16
case "int32":
bits = 32
case "int64":
bits = 64
}
n, err := strconv.ParseInt(payload, 10, bits)
if err != nil || strconv.FormatInt(n, 10) != payload {
return nil, false
}
switch tag {
case "int":
return int(n), true
case "int8":
return int8(n), true
case "int16":
return int16(n), true
case "int32":
return int32(n), true
default:
return n, true
}
case "string":
return payload, utf8.ValidString(payload)
case "time":
t, err := time.Parse(time.RFC3339Nano, payload)
t = t.UTC()
return t, err == nil && t.Year() >= 0 && t.Year() <= 9999
case "uuid":
id, err := uuid.Parse(payload)
return id, err == nil && id.String() == payload
default:
return nil, false
}
}
+220
View File
@@ -0,0 +1,220 @@
package page
import (
"encoding/base64"
"encoding/json"
"errors"
"math"
"reflect"
"strings"
"testing"
"time"
"uuid"
"github.com/stephenafamo/bob"
"github.com/stephenafamo/bob/dialect/psql"
pd "github.com/stephenafamo/bob/dialect/psql/dialect"
"github.com/stephenafamo/bob/expr"
)
func scalarSort() Sort {
return Sort{ID: "scalar/v1", Columns: []Column{{Expression: expr.Quote("id"), Unique: true}}}
}
func TestSupportedValues(t *testing.T) {
instant := time.Date(2026, 10, 27, 12, 34, 56, 123456789, time.FixedZone("odd", 3661))
values := []any{
int(0), int(math.MaxInt), int(math.MinInt), int8(math.MinInt8), int8(math.MaxInt8),
int16(math.MinInt16), int16(math.MaxInt16), int32(math.MinInt32), int32(math.MaxInt32),
int64(math.MinInt64), int64(math.MaxInt64), int64(9007199254740993), "", "こんにちは\x00' OR 1=1 --",
instant, time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(9999, 12, 31, 23, 59, 59, 999999999, time.UTC),
uuid.MustParse("01234567-89ab-cdef-0123-456789abcdef"), uuid.Nil(),
}
for _, value := range values {
t.Run(reflect.TypeOf(value).String(), func(t *testing.T) {
token, err := encodeCursor("scalar/v1", []any{value})
if err != nil {
t.Fatal(err)
}
decoded, err := decodeCursor(token, scalarSort())
if err != nil {
t.Fatal(err)
}
expected := value
if v, ok := value.(time.Time); ok {
expected = v.UTC()
}
if !reflect.DeepEqual(decoded, []any{expected}) {
t.Fatalf("type/value: got %#v want %#v", decoded, expected)
}
p, err := Prepare(scalarSort(), func(v any) []any { return []any{v} }, Request{Cursor: token}, Limits{Default: 1, Maximum: 2})
if err != nil {
t.Fatal(err)
}
q := psql.Select(Mod[*pd.SelectQuery](p))
_, args, err := bob.Build(t.Context(), q)
if err != nil || !reflect.DeepEqual(args, decoded) {
t.Fatalf("bound values: %#v %v", args, err)
}
if tm, ok := expected.(time.Time); ok {
data, _ := base64.RawURLEncoding.DecodeString(token)
if !strings.Contains(string(data), tm.Format(time.RFC3339Nano)) {
t.Fatalf("non-UTC timestamp: %s", data)
}
}
})
}
}
func TestUnsupportedValues(t *testing.T) {
type named int64
n := 1
values := []any{nil, named(1), uint(1), uint64(1), float64(1), true, &n, []byte("x"), struct{}{}, "\xff", time.Date(-1, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC)}
for _, value := range values {
p, err := Prepare(scalarSort(), func(v any) []any { return []any{v} }, Request{}, Limits{Default: 1, Maximum: 2})
if err != nil {
t.Fatal(err)
}
result, err := p.Page([]any{value})
var valueError *ValueError
if !errors.As(err, &valueError) || valueError.Position != 0 || !reflect.DeepEqual(result, Page[any]{}) {
t.Fatalf("%T: %+v %v", value, result, err)
}
}
type alias = int64
if _, err := encodeCursor("scalar/v1", []any{alias(1)}); err != nil {
t.Fatal(err)
}
}
func TestMalformedCursors(t *testing.T) {
valid := `{"v":1,"sort":"scalar/v1","values":[["int64","1"]]}`
cases := []struct{ data, kind string }{
{`null`, "malformed"}, {`[]`, "malformed"}, {`{`, "malformed"},
{valid + ` {}`, "malformed"},
{strings.Replace(valid, `"v":1`, `"v":2`, 1), "version"},
{strings.Replace(valid, `"v":1`, `"v":null`, 1), "malformed"},
{strings.Replace(valid, `"v":1`, `"v":1,"v":1`, 1), "malformed"},
{strings.Replace(valid, `"v":1,`, ``, 1), "malformed"},
{strings.Replace(valid, `"v":1`, `"v":1,"columns":[]`, 1), "malformed"},
{strings.Replace(valid, `scalar/v1`, `other/v1`, 1), "sort"},
{strings.Replace(valid, `"scalar/v1"`, `null`, 1), "malformed"},
{strings.Replace(valid, `[["int64","1"]]`, `null`, 1), "malformed"},
{strings.Replace(valid, `[["int64","1"]]`, `[]`, 1), "malformed"},
{strings.Replace(valid, `[["int64","1"]]`, `[["int64","1"],["int64","2"]]`, 1), "malformed"},
}
for _, entry := range []string{`null`, `{}`, `{"type":"int64","value":"1"}`, `[]`, `["int64"]`, `["int64","1","x"]`, `[null,"1"]`, `["int64",null]`, `["int64",1]`, `[1,"1"]`} {
cases = append(cases, struct{ data, kind string }{strings.Replace(valid, `["int64","1"]`, entry, 1), "malformed"})
}
for _, pair := range [][2]string{{"what", "1"}, {"int8", "128"}, {"int8", "-129"}, {"int16", "32768"}, {"int32", "2147483648"}, {"int64", "9223372036854775808"}, {"int", "999999999999999999999"}, {"int64", "01"}, {"int64", "+1"}, {"int64", "-0"}, {"int64", "1.0"}, {"time", "10000-01-01T00:00:00Z"}, {"time", "yesterday"}, {"time", "9999-12-31T23:59:59-01:00"}, {"time", "0000-01-01T00:00:00+01:00"}, {"uuid", "broken"}} {
data, _ := json.Marshal(pair)
cases = append(cases, struct{ data, kind string }{strings.Replace(valid, `["int64","1"]`, string(data), 1), "value"})
}
for i, tc := range cases {
_, err := decodeCursor(base64.RawURLEncoding.EncodeToString([]byte(tc.data)), scalarSort())
var ce *CursorError
if !errors.As(err, &ce) || ce.Kind != tc.kind {
t.Fatalf("case %d %s: %v", i, tc.data, err)
}
}
for _, token := range []string{"!", "====", base64.RawURLEncoding.EncodeToString([]byte(valid)) + "\n", base64.RawURLEncoding.EncodeToString([]byte("\xff"))} {
if _, err := decodeCursor(token, scalarSort()); err == nil {
t.Fatalf("accepted %q", token)
}
}
// A well-formed edit is a new seek boundary, not an authentication failure.
edited := strings.Replace(valid, `"1"`, `"9007199254740993"`, 1)
got, err := decodeCursor(base64.RawURLEncoding.EncodeToString([]byte(edited)), scalarSort())
if err != nil || got[0] != int64(9007199254740993) {
t.Fatalf("edited boundary: %v %v", got, err)
}
}
func TestCursorSize(t *testing.T) {
// Find the exact final-envelope boundary instead of assuming an overhead size.
var at string
var length int
for length = 12000; length < 12400; length++ {
token, err := encodeCursor("scalar/v1", []any{strings.Repeat("x", length)})
if err == nil && len(token) == maxCursorBytes {
at = token
break
}
}
if at == "" {
t.Fatal("exact boundary not found")
}
if _, err := decodeCursor(at, scalarSort()); err != nil {
t.Fatal(err)
}
var ce *CursorError
if _, err := decodeCursor(at+"A", scalarSort()); !errors.As(err, &ce) || ce.Kind != "size" {
t.Fatalf("received limit: %v", err)
}
p, err := Prepare(scalarSort(), func(v string) []any { return []any{v} }, Request{}, Limits{Default: 1, Maximum: 2})
if err != nil {
t.Fatal(err)
}
result, err := p.Page([]string{strings.Repeat("x", length+1)})
if !errors.As(err, &ce) || ce.Kind != "size" || !reflect.DeepEqual(result, Page[string]{}) {
t.Fatalf("emitted limit: %+v %v", result, err)
}
if _, err := encodeCursor("scalar/v1", []any{strings.Repeat("x", length-1)}); err != nil {
t.Fatal(err)
}
// Failure in an overflow row also invalidates the entire page.
result, err = p.Page([]string{"valid", strings.Repeat("x", length+1)})
if !errors.As(err, &ce) || !reflect.DeepEqual(result, Page[string]{}) {
t.Fatalf("overflow: %+v %v", result, err)
}
}
func TestPrepareAndKeyValidation(t *testing.T) {
limits := Limits{Default: 2, Maximum: 3}
badSorts := []Sort{{}, {ID: "x"}, {ID: "x", Columns: []Column{{Unique: true}}}, {ID: "x", Columns: []Column{{Expression: expr.Quote("id"), Direction: Order(2), Unique: true}}}, {ID: "x", Columns: []Column{{Expression: expr.Quote("id")}}}}
key := func(v int) []any { return []any{v} }
for _, sort := range badSorts {
if _, err := Prepare(sort, key, Request{}, limits); err == nil {
t.Fatalf("accepted sort: %+v", sort)
}
}
if _, err := Prepare[int](scalarSort(), nil, Request{}, limits); err == nil {
t.Fatal("nil key accepted")
}
for _, l := range []Limits{{}, {Default: 2, Maximum: 1}, {Default: 1, Maximum: math.MaxInt}} {
if _, err := Prepare(scalarSort(), key, Request{}, l); err == nil {
t.Fatalf("accepted limits %+v", l)
}
}
for _, r := range []Request{{Size: -1}, {Direction: Direction(2)}} {
if _, err := Prepare(scalarSort(), key, r, limits); err == nil {
t.Fatal("bad request accepted")
}
}
for _, size := range []int{0, 1, 3, 4, math.MaxInt} {
p, err := Prepare(scalarSort(), key, Request{Size: size}, limits)
want := min(size, 3)
if size == 0 {
want = 2
}
if err != nil || p.size != want {
t.Fatalf("size %d: %+v %v", size, p, err)
}
}
for _, badLength := range []int{0, 2} {
p, err := Prepare(scalarSort(), func(v int) []any {
if v == 9 {
return make([]any, badLength)
}
return []any{v}
}, Request{Size: 1}, limits)
if err != nil {
t.Fatal(err)
}
result, err := p.Page([]int{1, 9})
var ke *KeyError
if !errors.As(err, &ke) || ke.Expected != 1 || ke.Actual != badLength || !reflect.DeepEqual(result, Page[int]{}) {
t.Fatalf("key: %+v %v", result, err)
}
}
}
+69
View File
@@ -0,0 +1,69 @@
package page_test
import (
"context"
"errors"
"fmt"
"github.com/stephenafamo/bob"
"github.com/stephenafamo/bob/dialect/sqlite"
sd "github.com/stephenafamo/bob/dialect/sqlite/dialect"
"github.com/stephenafamo/bob/dialect/sqlite/sm"
"github.com/stephenafamo/bob/expr"
"github.com/stephenafamo/scan"
"go.auvem.com/corestack/page"
)
func Example() {
ctx := context.Background()
db, err := bob.Open("sqlite3", ":memory:")
if err != nil {
panic(err)
}
defer db.Close()
db.SetMaxOpenConns(1)
if _, err := db.ExecContext(ctx, "CREATE TABLE orders (id BIGINT PRIMARY KEY, priority BIGINT NOT NULL, tenant BIGINT NOT NULL)"); err != nil {
panic(err)
}
if _, err := db.ExecContext(ctx, "INSERT INTO orders VALUES (1,10,7),(2,10,7),(3,5,7),(4,20,8)"); err != nil {
panic(err)
}
type Order struct {
ID int64
Priority int64
}
sort := page.Sort{ID: "orders/priority/v1", Columns: []page.Column{
{Expression: expr.Quote("priority"), Direction: page.Desc},
{Expression: expr.Quote("id"), Direction: page.Asc, Unique: true},
}}
// With Bob-generated models, use their column references for these expressions.
// The key must independently match their order and actual stored value types.
key := func(o Order) []any { return []any{o.Priority, o.ID} }
p, err := page.Prepare(sort, key, page.Request{Size: 2}, page.Limits{Default: 2, Maximum: 100})
if err != nil {
panic(err)
}
q := sqlite.Select(sm.Columns("id", "priority"), sm.From("orders"), page.Mod[*sd.SelectQuery](p))
q.Apply(sm.Where(sqlite.Quote("tenant").EQ(sqlite.Arg(int64(7)))))
rows, err := bob.All(ctx, db, q, scan.StructMapper[Order]())
if err != nil {
panic(err)
}
result, err := p.Page(rows)
if err != nil {
panic(err)
}
fmt.Println(result.Items, result.HasNext, result.HasPrevious)
// Continue with Forward + result.EndCursor, or Backward + result.StartCursor.
// A mismatched key returns no partial Page, including for an overflow row.
bad, err := page.Prepare(sort, func(o Order) []any { return []any{o.ID} }, page.Request{}, page.Limits{Default: 2, Maximum: 100})
if err != nil {
panic(err)
}
_, err = bad.Page(rows)
var keyError *page.KeyError
fmt.Println(errors.As(err, &keyError), keyError.Expected, keyError.Actual)
// Output:
// [{1 10} {2 10}] true false
// true 2 1
}
+184
View File
@@ -0,0 +1,184 @@
// Package page provides transport-neutral keyset pagination for Bob SELECT queries.
// Callers own authorization, non-null stored sort values, and correspondence between
// Sort columns and the typed row key. The final column must itself be unique in the
// selected relation. Mixed directions are supported; null sort values are not.
//
// Sort IDs version the expressions, directions, key types and comparison semantics.
// Cursors expose values and allow edits; they are seek boundaries, not credentials.
// Concurrent writes or changed filters can skip or repeat rows between requests.
// Code generation may eventually align Sort and key; handwritten pairs have no
// compile-time field/type correspondence guarantee.
//
// Keys support exactly int, int8, int16, int32, int64, string, time.Time and the
// standard uuid.UUID. Named types, unsigned values, pointers and null wrappers
// require explicit caller conversion. Timestamps normalize to UTC with nanosecond
// precision. Versioned JSON cursors use unpadded base64url, limited to 16,384 bytes
// after encoding; unsupported values and malformed cursors return typed errors.
package page
import (
"fmt"
"slices"
"unicode/utf8"
"github.com/stephenafamo/bob"
)
// Order is a column's canonical ordering direction.
type Order uint8
const (
// Asc orders smaller values first.
Asc Order = iota
// Desc orders larger values first.
Desc
)
// Column declares a trusted, non-null SQL expression and its canonical direction.
// Unique asserts that this column alone is unique, including across joins.
type Column struct {
Expression bob.Expression
Direction Order
Unique bool
}
// Sort is an immutable, caller-owned ordering contract. Change ID when its
// expressions, order, key types, uniqueness or comparison semantics change.
type Sort struct {
ID string
Columns []Column
}
// Direction selects which side of an optional anchor to fetch.
type Direction uint8
const (
// Forward maps to first/after and is the default direction.
Forward Direction = iota
// Backward maps to last/before; returned items retain canonical sort order.
Backward
)
// Request carries a transport-independent page request. Zero Size uses the
// configured default; excessive sizes clamp to Maximum. Empty Cursor has no anchor.
type Request struct {
Direction Direction
Size int
Cursor string
}
// Limits configures page sizes; require 1 <= Default <= Maximum < max int.
type Limits struct {
Default int
Maximum int
}
// KeyError identifies a programmer error in the row key's arity.
type KeyError struct {
Expected int
Actual int
}
// Error implements error with the expected and actual key lengths.
func (e *KeyError) Error() string {
return fmt.Sprintf("page: key length %d, expected %d", e.Actual, e.Expected)
}
// Page contains canonical-order items and cursors for their first and last rows.
// Empty pages have empty cursors. Overflow determines the fetch-direction flag;
// anchor presence determines the opposite flag, even for empty/stale results.
// The opposite flag does not prove another row exists.
type Page[T any] struct {
Items []T
StartCursor string
EndCursor string
HasNext bool
HasPrevious bool
}
// Prepared holds validated pagination and the typed row key. Construct with Prepare.
// Treat Sort expressions and the key's captured state as immutable while in use.
type Prepared[T any] struct {
sort Sort
key func(T) []any
direction Direction
size int
anchor []any
}
// Prepare validates a request before a Bob mod can be applied. key must return
// exactly one supported concrete value per Sort column, in the same order, using
// actual stored/scanned values (including database timestamp precision).
func Prepare[T any](sort Sort, key func(T) []any, request Request, limits Limits) (*Prepared[T], error) {
if sort.ID == "" || !utf8.ValidString(sort.ID) || len(sort.Columns) == 0 {
return nil, fmt.Errorf("page: sort needs a valid ID and columns")
}
for i, c := range sort.Columns {
if c.Expression == nil || c.Direction > Desc {
return nil, fmt.Errorf("page: invalid sort column %d", i)
}
}
if !sort.Columns[len(sort.Columns)-1].Unique {
return nil, fmt.Errorf("page: final sort column must be unique")
}
if key == nil {
return nil, fmt.Errorf("page: nil key function")
}
if limits.Default < 1 || limits.Maximum < limits.Default || limits.Maximum == int(^uint(0)>>1) {
return nil, fmt.Errorf("page: invalid limits")
}
if request.Direction > Backward || request.Size < 0 {
return nil, fmt.Errorf("page: invalid request direction or size")
}
size := request.Size
if size == 0 {
size = limits.Default
}
size = min(size, limits.Maximum)
sort.Columns = slices.Clone(sort.Columns)
p := &Prepared[T]{sort: sort, key: key, direction: request.Direction, size: size}
if request.Cursor != "" {
var err error
p.anchor, err = decodeCursor(request.Cursor, sort)
if err != nil {
return nil, err
}
}
return p, nil
}
// Page validates all fetched rows, including overflow, and assembles a page.
// Pass rows in query order, with at most size+1 rows. It copies retained items;
// backward pagination never reverses the caller's slice. Errors return no page.
func (p *Prepared[T]) Page(rows []T) (Page[T], error) {
var zero Page[T]
if len(rows) > p.size+1 {
return zero, fmt.Errorf("page: too many fetched rows")
}
cursors := make([]string, len(rows))
for i, row := range rows {
key := p.key(row)
if len(key) != len(p.sort.Columns) {
return zero, &KeyError{Expected: len(p.sort.Columns), Actual: len(key)}
}
var err error
cursors[i], err = encodeCursor(p.sort.ID, key)
if err != nil {
return zero, err
}
}
overflow := len(rows) > p.size
n := min(len(rows), p.size)
result := Page[T]{Items: slices.Clone(rows[:n])}
if n > 0 {
result.StartCursor, result.EndCursor = cursors[0], cursors[n-1]
}
if p.direction == Backward {
slices.Reverse(result.Items)
result.StartCursor, result.EndCursor = result.EndCursor, result.StartCursor
result.HasPrevious, result.HasNext = overflow, len(p.anchor) > 0
} else {
result.HasNext, result.HasPrevious = overflow, len(p.anchor) > 0
}
return result, nil
}
+68
View File
@@ -0,0 +1,68 @@
package page
import (
"context"
"fmt"
"github.com/stephenafamo/bob"
"github.com/stephenafamo/bob/clause"
"github.com/stephenafamo/bob/expr"
"github.com/stephenafamo/scan"
)
// Mod applies validated keyset WHERE, ORDER BY and integer LIMIT size+1 to native
// PostgreSQL, SQLite or MySQL selects. Apply once to a query with no existing
// ORDER BY, LIMIT, OFFSET or FETCH; do not override pagination afterward.
// Callers may compose additional WHERE filters before or after this mod.
func Mod[Q interface {
AppendWhere(...any)
AppendOrder(bob.Expression)
SetLimit(any)
}, T any](p *Prepared[T]) bob.Mod[Q] {
return bob.ModFunc[Q](func(q Q) {
var alternatives []bob.Expression
for i, c := range p.sort.Columns {
desc := c.Direction == Desc
if p.direction == Backward {
desc = !desc
}
direction, operator := "ASC", ">"
if desc {
direction, operator = "DESC", "<"
}
q.AppendOrder(clause.OrderDef{Expression: c.Expression, Direction: direction})
if len(p.anchor) == 0 {
continue
}
terms := make([]bob.Expression, 0, i+1)
for j := 0; j < i; j++ {
terms = append(terms, expr.OP("=", p.sort.Columns[j].Expression, expr.Arg(p.anchor[j])))
}
terms = append(terms, expr.OP(operator, c.Expression, expr.Arg(p.anchor[i])))
alternatives = append(alternatives, grouped(terms, " AND "))
}
// authz: group the seek disjunction so caller filters constrain every branch.
if len(alternatives) > 0 {
q.AppendWhere(grouped(alternatives, " OR "))
}
q.SetLimit(p.size + 1)
})
}
func grouped(expressions []bob.Expression, separator string) bob.Expression {
return expr.Join{Exprs: []bob.Expression{
expr.Raw("("), expr.Join{Exprs: expressions, Sep: separator}, expr.Raw(")"),
}, Sep: expr.NoSep}
}
// Count executes a separate caller-built, authorized count query returning one
// int64 column. It never rewrites a paginated query or infers grouped/distinct
// semantics. Nothing runs unless called.
func Count(ctx context.Context, executor bob.Executor, query bob.Query) (int64, error) {
// authz: callers supply the same authorized relation as their item query.
count, err := bob.One(ctx, executor, query, scan.SingleColumnMapper[int64])
if err != nil {
return 0, fmt.Errorf("page: count: %w", err)
}
return count, nil
}
+189
View File
@@ -0,0 +1,189 @@
package page_test
import (
"database/sql"
"os"
"reflect"
"errors"
"testing"
_ "github.com/lib/pq"
_ "github.com/mattn/go-sqlite3"
"github.com/stephenafamo/bob"
"github.com/stephenafamo/bob/dialect/mysql"
md "github.com/stephenafamo/bob/dialect/mysql/dialect"
mm "github.com/stephenafamo/bob/dialect/mysql/sm"
"github.com/stephenafamo/bob/dialect/psql"
pd "github.com/stephenafamo/bob/dialect/psql/dialect"
pm "github.com/stephenafamo/bob/dialect/psql/sm"
"github.com/stephenafamo/bob/dialect/sqlite"
sd "github.com/stephenafamo/bob/dialect/sqlite/dialect"
"github.com/stephenafamo/bob/dialect/sqlite/sm"
"github.com/stephenafamo/bob/expr"
"github.com/stephenafamo/scan"
"go.auvem.com/corestack/page"
)
type item struct {
ID int64
Score int64
}
func itemSort() page.Sort {
return page.Sort{ID: "items/score/v1", Columns: []page.Column{
{Expression: expr.Quote("score"), Direction: page.Desc},
{Expression: expr.Quote("id"), Direction: page.Asc, Unique: true},
}}
}
func itemKey(i item) []any { return []any{i.Score, i.ID} }
func TestSQLiteFlow(t *testing.T) {
db, err := bob.Open("sqlite3", ":memory:")
if err != nil {
t.Fatal(err)
}
defer db.Close()
databaseFlow(t, db, false)
}
func TestPostgresFlow(t *testing.T) {
dsn := os.Getenv("PAGE_POSTGRES_DSN")
if dsn == "" {
if os.Getenv("PAGE_REQUIRE_POSTGRES") != "" {
t.Fatal("PAGE_POSTGRES_DSN is required")
}
t.Skip("set PAGE_POSTGRES_DSN to run PostgreSQL integration")
}
db, err := bob.Open("postgres", dsn)
if err != nil {
t.Fatal(err)
}
defer db.Close()
databaseFlow(t, db, true)
}
func databaseFlow(t *testing.T, db bob.DB, postgres bool) {
t.Helper()
db.SetMaxOpenConns(1)
for _, statement := range []string{
"CREATE TEMP TABLE items (id BIGINT PRIMARY KEY, score BIGINT NOT NULL, tenant BIGINT NOT NULL)",
"INSERT INTO items VALUES (1,30,1),(2,30,1),(3,20,1),(4,10,1),(5,5,2),(6,30,2)",
} {
if _, err := db.ExecContext(t.Context(), statement); err != nil {
t.Fatal(err)
}
}
fetch := func(direction page.Direction, size int, cursor string) page.Page[item] {
t.Helper()
p, err := page.Prepare(itemSort(), itemKey, page.Request{Direction: direction, Size: size, Cursor: cursor}, page.Limits{Default: 2, Maximum: 3})
if err != nil {
t.Fatal(err)
}
var q bob.Query
if postgres {
query := psql.Select(pm.Columns("id", "score"), pm.From("items"), pm.Where(psql.Quote("tenant").EQ(psql.Arg(int64(1)))))
query.Apply(page.Mod[*pd.SelectQuery](p))
query.Apply(pm.Where(psql.Quote("score").GT(psql.Arg(int64(0)))))
q = query
} else {
query := sqlite.Select(sm.Columns("id", "score"), sm.From("items"), sm.Where(sqlite.Quote("tenant").EQ(sqlite.Arg(int64(1)))))
query.Apply(page.Mod[*sd.SelectQuery](p))
query.Apply(sm.Where(sqlite.Quote("score").GT(sqlite.Arg(int64(0)))))
q = query
}
rows, err := bob.All(t.Context(), db, q, scan.StructMapper[item]())
if err != nil {
t.Fatal(err)
}
before := append([]item(nil), rows...)
result, err := p.Page(rows)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(before, rows) {
t.Fatal("Page mutated fetched slice")
}
return result
}
check := func(got page.Page[item], ids []int64, next, previous bool) {
t.Helper()
actual := make([]int64, len(got.Items))
for i, row := range got.Items {
actual[i] = row.ID
}
if !reflect.DeepEqual(actual, ids) || got.HasNext != next || got.HasPrevious != previous {
t.Fatalf("got IDs %v next=%v previous=%v; want %v %v %v", actual, got.HasNext, got.HasPrevious, ids, next, previous)
}
if len(ids) == 0 && (got.StartCursor != "" || got.EndCursor != "") {
t.Fatal("empty page has cursors")
}
if len(ids) > 0 && (got.StartCursor == "" || got.EndCursor == "") {
t.Fatal("missing cursors")
}
}
first := fetch(page.Forward, 2, "")
check(first, []int64{1, 2}, true, false)
second := fetch(page.Forward, 2, first.EndCursor)
check(second, []int64{3, 4}, false, true)
check(fetch(page.Forward, 2, second.EndCursor), []int64{}, false, true)
last := fetch(page.Backward, 2, "")
check(last, []int64{3, 4}, false, true)
previous := fetch(page.Backward, 2, last.StartCursor)
check(previous, []int64{1, 2}, true, false)
check(fetch(page.Backward, 2, previous.StartCursor), []int64{}, true, false)
check(fetch(page.Forward, 0, ""), []int64{1, 2}, true, false)
check(fetch(page.Forward, 1000, ""), []int64{1, 2, 3}, true, false)
// A cursor describes a value boundary even after its row is deleted.
if _, err := db.ExecContext(t.Context(), "DELETE FROM items WHERE id=2"); err != nil {
t.Fatal(err)
}
check(fetch(page.Forward, 2, first.EndCursor), []int64{3, 4}, false, true)
var countQuery, emptyQuery bob.Query
if postgres {
countQuery = psql.Select(pm.Columns("count(*)"), pm.From("items"), pm.Where(psql.Quote("tenant").EQ(psql.Arg(1))))
emptyQuery = psql.Select(pm.Columns("id"), pm.From("items"), pm.Where(psql.Quote("id").LT(psql.Arg(0))))
} else {
countQuery = sqlite.Select(sm.Columns("count(*)"), sm.From("items"), sm.Where(sqlite.Quote("tenant").EQ(sqlite.Arg(1))))
emptyQuery = sqlite.Select(sm.Columns("id"), sm.From("items"), sm.Where(sqlite.Quote("id").LT(sqlite.Arg(0))))
}
count, err := page.Count(t.Context(), db, countQuery)
if err != nil || count != 3 {
t.Fatalf("count %d: %v", count, err)
}
if _, err := page.Count(t.Context(), db, emptyQuery); !errors.Is(err, sql.ErrNoRows) {
t.Fatalf("count error lost cause: %v", err)
}
_, err = page.Prepare(itemSort(), itemKey, page.Request{Cursor: "not!base64"}, page.Limits{Default: 2, Maximum: 3})
var cursorError *page.CursorError
if !errors.As(err, &cursorError) {
t.Fatalf("invalid cursor: %v", err)
}
bad, err := page.Prepare(itemSort(), func(item) []any { return []any{int64(1)} }, page.Request{}, page.Limits{Default: 2, Maximum: 3})
if err != nil {
t.Fatal(err)
}
result, err := bad.Page([]item{{ID: 1}})
var keyError *page.KeyError
if !errors.As(err, &keyError) || !reflect.DeepEqual(result, page.Page[item]{}) {
t.Fatalf("key failure: %+v %v", result, err)
}
if _, err := db.ExecContext(t.Context(), "DELETE FROM items"); err != nil {
t.Fatal(err)
}
check(fetch(page.Forward, 2, ""), []int64{}, false, false)
check(fetch(page.Backward, 2, ""), []int64{}, false, false)
check(fetch(page.Forward, 2, first.EndCursor), []int64{}, false, true)
check(fetch(page.Backward, 2, last.StartCursor), []int64{}, true, false)
}
func TestMySQLModComposition(t *testing.T) {
p, err := page.Prepare(itemSort(), itemKey, page.Request{}, page.Limits{Default: 2, Maximum: 3})
if err != nil {
t.Fatal(err)
}
q := mysql.Select(mm.Columns("id", "score"), mm.From("items"), page.Mod[*md.SelectQuery](p))
q.Apply(mm.Where(mysql.Quote("tenant").EQ(mysql.Arg(int64(1)))))
// Compile-time composition only; MySQL runtime behavior needs a database test.
}