feat(page): add typed keyset pagination for Bob
Add versioned typed cursors, composable native query mods, bidirectional page assembly and opt-in authorized counts. Cover SQLite and PostgreSQL outcomes, strict cursor boundaries and a runnable example; require PostgreSQL service tests in CI. Refs: #6
This commit is contained in:
1 parent
da1ff94fe2
commit
8615d8ca66
11 files changed
+1142
-11
No files matched your search
+216
@@ -0,0 +1,216 @@
|
||||
package page
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
"uuid"
|
||||
)
|
||||
|
||||
const maxCursorBytes = 16384
|
||||
|
||||
// CursorError identifies malformed, version, sort, value, or size cursor failures via
|
||||
// Kind. It intentionally omits cursor payloads from its message.
|
||||
type CursorError struct{ Kind string }
|
||||
|
||||
// Error implements error without including cursor contents.
|
||||
func (e *CursorError) Error() string { return "page: cursor " + e.Kind }
|
||||
|
||||
// ValueError identifies an unsupported concrete type or invalid supported value
|
||||
// at Position in the row key, without exposing the value itself.
|
||||
type ValueError struct {
|
||||
Position int
|
||||
Type string
|
||||
}
|
||||
|
||||
// Error implements error without including key values.
|
||||
func (e *ValueError) Error() string {
|
||||
return fmt.Sprintf("page: invalid key value at %d (%s)", e.Position, e.Type)
|
||||
}
|
||||
|
||||
type envelope struct {
|
||||
Version int `json:"v"`
|
||||
Sort string `json:"sort"`
|
||||
Values [][2]string `json:"values"`
|
||||
}
|
||||
|
||||
func encodeCursor(sortID string, values []any) (string, error) {
|
||||
e := envelope{Version: 1, Sort: sortID, Values: make([][2]string, len(values))}
|
||||
for i, v := range values {
|
||||
pair, ok := encodeValue(v)
|
||||
if !ok {
|
||||
return "", &ValueError{Position: i, Type: fmt.Sprintf("%T", v)}
|
||||
}
|
||||
e.Values[i] = pair
|
||||
}
|
||||
data, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
return "", &CursorError{Kind: "malformed"}
|
||||
}
|
||||
if base64.RawURLEncoding.EncodedLen(len(data)) > maxCursorBytes {
|
||||
return "", &CursorError{Kind: "size"}
|
||||
}
|
||||
return base64.RawURLEncoding.EncodeToString(data), nil
|
||||
}
|
||||
|
||||
func encodeValue(value any) ([2]string, bool) {
|
||||
var tag, payload string
|
||||
switch v := value.(type) {
|
||||
case int:
|
||||
tag, payload = "int", strconv.FormatInt(int64(v), 10)
|
||||
case int8:
|
||||
tag, payload = "int8", strconv.FormatInt(int64(v), 10)
|
||||
case int16:
|
||||
tag, payload = "int16", strconv.FormatInt(int64(v), 10)
|
||||
case int32:
|
||||
tag, payload = "int32", strconv.FormatInt(int64(v), 10)
|
||||
case int64:
|
||||
tag, payload = "int64", strconv.FormatInt(v, 10)
|
||||
case string:
|
||||
if !utf8.ValidString(v) {
|
||||
return [2]string{}, false
|
||||
}
|
||||
tag, payload = "string", v
|
||||
case time.Time:
|
||||
v = v.UTC()
|
||||
if v.Year() < 0 || v.Year() > 9999 {
|
||||
return [2]string{}, false
|
||||
}
|
||||
tag, payload = "time", v.Format(time.RFC3339Nano)
|
||||
case uuid.UUID:
|
||||
tag, payload = "uuid", v.String()
|
||||
default:
|
||||
return [2]string{}, false
|
||||
}
|
||||
return [2]string{tag, payload}, true
|
||||
}
|
||||
|
||||
func decodeCursor(token string, sort Sort) ([]any, error) {
|
||||
failure := func(kind string) ([]any, error) { return nil, &CursorError{Kind: kind} }
|
||||
if len(token) > maxCursorBytes {
|
||||
return failure("size")
|
||||
}
|
||||
data, err := base64.RawURLEncoding.Strict().DecodeString(token)
|
||||
if err != nil || !utf8.Valid(data) || base64.RawURLEncoding.EncodeToString(data) != token {
|
||||
return failure("malformed")
|
||||
}
|
||||
// invariant: exactly three distinct envelope fields; struct decoding alone permits duplicates.
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
start, err := decoder.Token()
|
||||
if err != nil || start != json.Delim('{') {
|
||||
return failure("malformed")
|
||||
}
|
||||
fields := make(map[string]json.RawMessage, 3)
|
||||
for decoder.More() {
|
||||
name, err := decoder.Token()
|
||||
if err != nil {
|
||||
return failure("malformed")
|
||||
}
|
||||
key, ok := name.(string)
|
||||
if !ok || (key != "v" && key != "sort" && key != "values") {
|
||||
return failure("malformed")
|
||||
}
|
||||
if _, exists := fields[key]; exists {
|
||||
return failure("malformed")
|
||||
}
|
||||
var raw json.RawMessage
|
||||
if decoder.Decode(&raw) != nil {
|
||||
return failure("malformed")
|
||||
}
|
||||
fields[key] = raw
|
||||
}
|
||||
if _, err := decoder.Token(); err != nil || len(fields) != 3 {
|
||||
return failure("malformed")
|
||||
}
|
||||
if _, err := decoder.Token(); err != io.EOF {
|
||||
return failure("malformed")
|
||||
}
|
||||
var version int
|
||||
if bytes.Equal(fields["v"], []byte("null")) || json.Unmarshal(fields["v"], &version) != nil {
|
||||
return failure("malformed")
|
||||
}
|
||||
if version != 1 {
|
||||
return failure("version")
|
||||
}
|
||||
var id string
|
||||
if !jsonString(fields["sort"], &id) {
|
||||
return failure("malformed")
|
||||
}
|
||||
if id != sort.ID {
|
||||
return failure("sort")
|
||||
}
|
||||
var entries []json.RawMessage
|
||||
if len(fields["values"]) == 0 || fields["values"][0] != '[' || json.Unmarshal(fields["values"], &entries) != nil || len(entries) != len(sort.Columns) {
|
||||
return failure("malformed")
|
||||
}
|
||||
values := make([]any, len(entries))
|
||||
for i, entry := range entries {
|
||||
var pair []json.RawMessage
|
||||
if len(entry) == 0 || entry[0] != '[' || json.Unmarshal(entry, &pair) != nil || len(pair) != 2 {
|
||||
return failure("malformed")
|
||||
}
|
||||
var tag, payload string
|
||||
if !jsonString(pair[0], &tag) || !jsonString(pair[1], &payload) {
|
||||
return failure("malformed")
|
||||
}
|
||||
value, ok := decodeValue(tag, payload)
|
||||
if !ok {
|
||||
return failure("value")
|
||||
}
|
||||
values[i] = value
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
func jsonString(raw json.RawMessage, value *string) bool {
|
||||
return len(raw) > 0 && raw[0] == '"' && json.Unmarshal(raw, value) == nil && utf8.ValidString(*value)
|
||||
}
|
||||
|
||||
func decodeValue(tag, payload string) (any, bool) {
|
||||
switch tag {
|
||||
case "int", "int8", "int16", "int32", "int64":
|
||||
bits := strconv.IntSize
|
||||
switch tag {
|
||||
case "int8":
|
||||
bits = 8
|
||||
case "int16":
|
||||
bits = 16
|
||||
case "int32":
|
||||
bits = 32
|
||||
case "int64":
|
||||
bits = 64
|
||||
}
|
||||
n, err := strconv.ParseInt(payload, 10, bits)
|
||||
if err != nil || strconv.FormatInt(n, 10) != payload {
|
||||
return nil, false
|
||||
}
|
||||
switch tag {
|
||||
case "int":
|
||||
return int(n), true
|
||||
case "int8":
|
||||
return int8(n), true
|
||||
case "int16":
|
||||
return int16(n), true
|
||||
case "int32":
|
||||
return int32(n), true
|
||||
default:
|
||||
return n, true
|
||||
}
|
||||
case "string":
|
||||
return payload, utf8.ValidString(payload)
|
||||
case "time":
|
||||
t, err := time.Parse(time.RFC3339Nano, payload)
|
||||
t = t.UTC()
|
||||
return t, err == nil && t.Year() >= 0 && t.Year() <= 9999
|
||||
case "uuid":
|
||||
id, err := uuid.Parse(payload)
|
||||
return id, err == nil && id.String() == payload
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,220 @@
|
||||
package page
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
"uuid"
|
||||
|
||||
"github.com/stephenafamo/bob"
|
||||
"github.com/stephenafamo/bob/dialect/psql"
|
||||
pd "github.com/stephenafamo/bob/dialect/psql/dialect"
|
||||
"github.com/stephenafamo/bob/expr"
|
||||
)
|
||||
|
||||
func scalarSort() Sort {
|
||||
return Sort{ID: "scalar/v1", Columns: []Column{{Expression: expr.Quote("id"), Unique: true}}}
|
||||
}
|
||||
|
||||
func TestSupportedValues(t *testing.T) {
|
||||
instant := time.Date(2026, 10, 27, 12, 34, 56, 123456789, time.FixedZone("odd", 3661))
|
||||
values := []any{
|
||||
int(0), int(math.MaxInt), int(math.MinInt), int8(math.MinInt8), int8(math.MaxInt8),
|
||||
int16(math.MinInt16), int16(math.MaxInt16), int32(math.MinInt32), int32(math.MaxInt32),
|
||||
int64(math.MinInt64), int64(math.MaxInt64), int64(9007199254740993), "", "こんにちは\x00' OR 1=1 --",
|
||||
instant, time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(9999, 12, 31, 23, 59, 59, 999999999, time.UTC),
|
||||
uuid.MustParse("01234567-89ab-cdef-0123-456789abcdef"), uuid.Nil(),
|
||||
}
|
||||
for _, value := range values {
|
||||
t.Run(reflect.TypeOf(value).String(), func(t *testing.T) {
|
||||
token, err := encodeCursor("scalar/v1", []any{value})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
decoded, err := decodeCursor(token, scalarSort())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expected := value
|
||||
if v, ok := value.(time.Time); ok {
|
||||
expected = v.UTC()
|
||||
}
|
||||
if !reflect.DeepEqual(decoded, []any{expected}) {
|
||||
t.Fatalf("type/value: got %#v want %#v", decoded, expected)
|
||||
}
|
||||
p, err := Prepare(scalarSort(), func(v any) []any { return []any{v} }, Request{Cursor: token}, Limits{Default: 1, Maximum: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q := psql.Select(Mod[*pd.SelectQuery](p))
|
||||
_, args, err := bob.Build(t.Context(), q)
|
||||
if err != nil || !reflect.DeepEqual(args, decoded) {
|
||||
t.Fatalf("bound values: %#v %v", args, err)
|
||||
}
|
||||
if tm, ok := expected.(time.Time); ok {
|
||||
data, _ := base64.RawURLEncoding.DecodeString(token)
|
||||
if !strings.Contains(string(data), tm.Format(time.RFC3339Nano)) {
|
||||
t.Fatalf("non-UTC timestamp: %s", data)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsupportedValues(t *testing.T) {
|
||||
type named int64
|
||||
n := 1
|
||||
values := []any{nil, named(1), uint(1), uint64(1), float64(1), true, &n, []byte("x"), struct{}{}, "\xff", time.Date(-1, 1, 1, 0, 0, 0, 0, time.UTC), time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC)}
|
||||
for _, value := range values {
|
||||
p, err := Prepare(scalarSort(), func(v any) []any { return []any{v} }, Request{}, Limits{Default: 1, Maximum: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := p.Page([]any{value})
|
||||
var valueError *ValueError
|
||||
if !errors.As(err, &valueError) || valueError.Position != 0 || !reflect.DeepEqual(result, Page[any]{}) {
|
||||
t.Fatalf("%T: %+v %v", value, result, err)
|
||||
}
|
||||
}
|
||||
type alias = int64
|
||||
if _, err := encodeCursor("scalar/v1", []any{alias(1)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMalformedCursors(t *testing.T) {
|
||||
valid := `{"v":1,"sort":"scalar/v1","values":[["int64","1"]]}`
|
||||
cases := []struct{ data, kind string }{
|
||||
{`null`, "malformed"}, {`[]`, "malformed"}, {`{`, "malformed"},
|
||||
{valid + ` {}`, "malformed"},
|
||||
{strings.Replace(valid, `"v":1`, `"v":2`, 1), "version"},
|
||||
{strings.Replace(valid, `"v":1`, `"v":null`, 1), "malformed"},
|
||||
{strings.Replace(valid, `"v":1`, `"v":1,"v":1`, 1), "malformed"},
|
||||
{strings.Replace(valid, `"v":1,`, ``, 1), "malformed"},
|
||||
{strings.Replace(valid, `"v":1`, `"v":1,"columns":[]`, 1), "malformed"},
|
||||
{strings.Replace(valid, `scalar/v1`, `other/v1`, 1), "sort"},
|
||||
{strings.Replace(valid, `"scalar/v1"`, `null`, 1), "malformed"},
|
||||
{strings.Replace(valid, `[["int64","1"]]`, `null`, 1), "malformed"},
|
||||
{strings.Replace(valid, `[["int64","1"]]`, `[]`, 1), "malformed"},
|
||||
{strings.Replace(valid, `[["int64","1"]]`, `[["int64","1"],["int64","2"]]`, 1), "malformed"},
|
||||
}
|
||||
for _, entry := range []string{`null`, `{}`, `{"type":"int64","value":"1"}`, `[]`, `["int64"]`, `["int64","1","x"]`, `[null,"1"]`, `["int64",null]`, `["int64",1]`, `[1,"1"]`} {
|
||||
cases = append(cases, struct{ data, kind string }{strings.Replace(valid, `["int64","1"]`, entry, 1), "malformed"})
|
||||
}
|
||||
for _, pair := range [][2]string{{"what", "1"}, {"int8", "128"}, {"int8", "-129"}, {"int16", "32768"}, {"int32", "2147483648"}, {"int64", "9223372036854775808"}, {"int", "999999999999999999999"}, {"int64", "01"}, {"int64", "+1"}, {"int64", "-0"}, {"int64", "1.0"}, {"time", "10000-01-01T00:00:00Z"}, {"time", "yesterday"}, {"time", "9999-12-31T23:59:59-01:00"}, {"time", "0000-01-01T00:00:00+01:00"}, {"uuid", "broken"}} {
|
||||
data, _ := json.Marshal(pair)
|
||||
cases = append(cases, struct{ data, kind string }{strings.Replace(valid, `["int64","1"]`, string(data), 1), "value"})
|
||||
}
|
||||
for i, tc := range cases {
|
||||
_, err := decodeCursor(base64.RawURLEncoding.EncodeToString([]byte(tc.data)), scalarSort())
|
||||
var ce *CursorError
|
||||
if !errors.As(err, &ce) || ce.Kind != tc.kind {
|
||||
t.Fatalf("case %d %s: %v", i, tc.data, err)
|
||||
}
|
||||
}
|
||||
for _, token := range []string{"!", "====", base64.RawURLEncoding.EncodeToString([]byte(valid)) + "\n", base64.RawURLEncoding.EncodeToString([]byte("\xff"))} {
|
||||
if _, err := decodeCursor(token, scalarSort()); err == nil {
|
||||
t.Fatalf("accepted %q", token)
|
||||
}
|
||||
}
|
||||
// A well-formed edit is a new seek boundary, not an authentication failure.
|
||||
edited := strings.Replace(valid, `"1"`, `"9007199254740993"`, 1)
|
||||
got, err := decodeCursor(base64.RawURLEncoding.EncodeToString([]byte(edited)), scalarSort())
|
||||
if err != nil || got[0] != int64(9007199254740993) {
|
||||
t.Fatalf("edited boundary: %v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCursorSize(t *testing.T) {
|
||||
// Find the exact final-envelope boundary instead of assuming an overhead size.
|
||||
var at string
|
||||
var length int
|
||||
for length = 12000; length < 12400; length++ {
|
||||
token, err := encodeCursor("scalar/v1", []any{strings.Repeat("x", length)})
|
||||
if err == nil && len(token) == maxCursorBytes {
|
||||
at = token
|
||||
break
|
||||
}
|
||||
}
|
||||
if at == "" {
|
||||
t.Fatal("exact boundary not found")
|
||||
}
|
||||
if _, err := decodeCursor(at, scalarSort()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var ce *CursorError
|
||||
if _, err := decodeCursor(at+"A", scalarSort()); !errors.As(err, &ce) || ce.Kind != "size" {
|
||||
t.Fatalf("received limit: %v", err)
|
||||
}
|
||||
p, err := Prepare(scalarSort(), func(v string) []any { return []any{v} }, Request{}, Limits{Default: 1, Maximum: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := p.Page([]string{strings.Repeat("x", length+1)})
|
||||
if !errors.As(err, &ce) || ce.Kind != "size" || !reflect.DeepEqual(result, Page[string]{}) {
|
||||
t.Fatalf("emitted limit: %+v %v", result, err)
|
||||
}
|
||||
if _, err := encodeCursor("scalar/v1", []any{strings.Repeat("x", length-1)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Failure in an overflow row also invalidates the entire page.
|
||||
result, err = p.Page([]string{"valid", strings.Repeat("x", length+1)})
|
||||
if !errors.As(err, &ce) || !reflect.DeepEqual(result, Page[string]{}) {
|
||||
t.Fatalf("overflow: %+v %v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareAndKeyValidation(t *testing.T) {
|
||||
limits := Limits{Default: 2, Maximum: 3}
|
||||
badSorts := []Sort{{}, {ID: "x"}, {ID: "x", Columns: []Column{{Unique: true}}}, {ID: "x", Columns: []Column{{Expression: expr.Quote("id"), Direction: Order(2), Unique: true}}}, {ID: "x", Columns: []Column{{Expression: expr.Quote("id")}}}}
|
||||
key := func(v int) []any { return []any{v} }
|
||||
for _, sort := range badSorts {
|
||||
if _, err := Prepare(sort, key, Request{}, limits); err == nil {
|
||||
t.Fatalf("accepted sort: %+v", sort)
|
||||
}
|
||||
}
|
||||
if _, err := Prepare[int](scalarSort(), nil, Request{}, limits); err == nil {
|
||||
t.Fatal("nil key accepted")
|
||||
}
|
||||
for _, l := range []Limits{{}, {Default: 2, Maximum: 1}, {Default: 1, Maximum: math.MaxInt}} {
|
||||
if _, err := Prepare(scalarSort(), key, Request{}, l); err == nil {
|
||||
t.Fatalf("accepted limits %+v", l)
|
||||
}
|
||||
}
|
||||
for _, r := range []Request{{Size: -1}, {Direction: Direction(2)}} {
|
||||
if _, err := Prepare(scalarSort(), key, r, limits); err == nil {
|
||||
t.Fatal("bad request accepted")
|
||||
}
|
||||
}
|
||||
for _, size := range []int{0, 1, 3, 4, math.MaxInt} {
|
||||
p, err := Prepare(scalarSort(), key, Request{Size: size}, limits)
|
||||
want := min(size, 3)
|
||||
if size == 0 {
|
||||
want = 2
|
||||
}
|
||||
if err != nil || p.size != want {
|
||||
t.Fatalf("size %d: %+v %v", size, p, err)
|
||||
}
|
||||
}
|
||||
for _, badLength := range []int{0, 2} {
|
||||
p, err := Prepare(scalarSort(), func(v int) []any {
|
||||
if v == 9 {
|
||||
return make([]any, badLength)
|
||||
}
|
||||
return []any{v}
|
||||
}, Request{Size: 1}, limits)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := p.Page([]int{1, 9})
|
||||
var ke *KeyError
|
||||
if !errors.As(err, &ke) || ke.Expected != 1 || ke.Actual != badLength || !reflect.DeepEqual(result, Page[int]{}) {
|
||||
t.Fatalf("key: %+v %v", result, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
package page_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/stephenafamo/bob"
|
||||
"github.com/stephenafamo/bob/dialect/sqlite"
|
||||
sd "github.com/stephenafamo/bob/dialect/sqlite/dialect"
|
||||
"github.com/stephenafamo/bob/dialect/sqlite/sm"
|
||||
"github.com/stephenafamo/bob/expr"
|
||||
"github.com/stephenafamo/scan"
|
||||
"go.auvem.com/corestack/page"
|
||||
)
|
||||
|
||||
func Example() {
|
||||
ctx := context.Background()
|
||||
db, err := bob.Open("sqlite3", ":memory:")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer db.Close()
|
||||
db.SetMaxOpenConns(1)
|
||||
if _, err := db.ExecContext(ctx, "CREATE TABLE orders (id BIGINT PRIMARY KEY, priority BIGINT NOT NULL, tenant BIGINT NOT NULL)"); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if _, err := db.ExecContext(ctx, "INSERT INTO orders VALUES (1,10,7),(2,10,7),(3,5,7),(4,20,8)"); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
type Order struct {
|
||||
ID int64
|
||||
Priority int64
|
||||
}
|
||||
sort := page.Sort{ID: "orders/priority/v1", Columns: []page.Column{
|
||||
{Expression: expr.Quote("priority"), Direction: page.Desc},
|
||||
{Expression: expr.Quote("id"), Direction: page.Asc, Unique: true},
|
||||
}}
|
||||
// With Bob-generated models, use their column references for these expressions.
|
||||
// The key must independently match their order and actual stored value types.
|
||||
key := func(o Order) []any { return []any{o.Priority, o.ID} }
|
||||
p, err := page.Prepare(sort, key, page.Request{Size: 2}, page.Limits{Default: 2, Maximum: 100})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
q := sqlite.Select(sm.Columns("id", "priority"), sm.From("orders"), page.Mod[*sd.SelectQuery](p))
|
||||
q.Apply(sm.Where(sqlite.Quote("tenant").EQ(sqlite.Arg(int64(7)))))
|
||||
rows, err := bob.All(ctx, db, q, scan.StructMapper[Order]())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
result, err := p.Page(rows)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(result.Items, result.HasNext, result.HasPrevious)
|
||||
// Continue with Forward + result.EndCursor, or Backward + result.StartCursor.
|
||||
// A mismatched key returns no partial Page, including for an overflow row.
|
||||
bad, err := page.Prepare(sort, func(o Order) []any { return []any{o.ID} }, page.Request{}, page.Limits{Default: 2, Maximum: 100})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
_, err = bad.Page(rows)
|
||||
var keyError *page.KeyError
|
||||
fmt.Println(errors.As(err, &keyError), keyError.Expected, keyError.Actual)
|
||||
// Output:
|
||||
// [{1 10} {2 10}] true false
|
||||
// true 2 1
|
||||
}
|
||||
+184
@@ -0,0 +1,184 @@
|
||||
// Package page provides transport-neutral keyset pagination for Bob SELECT queries.
|
||||
// Callers own authorization, non-null stored sort values, and correspondence between
|
||||
// Sort columns and the typed row key. The final column must itself be unique in the
|
||||
// selected relation. Mixed directions are supported; null sort values are not.
|
||||
//
|
||||
// Sort IDs version the expressions, directions, key types and comparison semantics.
|
||||
// Cursors expose values and allow edits; they are seek boundaries, not credentials.
|
||||
// Concurrent writes or changed filters can skip or repeat rows between requests.
|
||||
// Code generation may eventually align Sort and key; handwritten pairs have no
|
||||
// compile-time field/type correspondence guarantee.
|
||||
//
|
||||
// Keys support exactly int, int8, int16, int32, int64, string, time.Time and the
|
||||
// standard uuid.UUID. Named types, unsigned values, pointers and null wrappers
|
||||
// require explicit caller conversion. Timestamps normalize to UTC with nanosecond
|
||||
// precision. Versioned JSON cursors use unpadded base64url, limited to 16,384 bytes
|
||||
// after encoding; unsupported values and malformed cursors return typed errors.
|
||||
package page
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/stephenafamo/bob"
|
||||
)
|
||||
|
||||
// Order is a column's canonical ordering direction.
|
||||
type Order uint8
|
||||
|
||||
const (
|
||||
// Asc orders smaller values first.
|
||||
Asc Order = iota
|
||||
// Desc orders larger values first.
|
||||
Desc
|
||||
)
|
||||
|
||||
// Column declares a trusted, non-null SQL expression and its canonical direction.
|
||||
// Unique asserts that this column alone is unique, including across joins.
|
||||
type Column struct {
|
||||
Expression bob.Expression
|
||||
Direction Order
|
||||
Unique bool
|
||||
}
|
||||
|
||||
// Sort is an immutable, caller-owned ordering contract. Change ID when its
|
||||
// expressions, order, key types, uniqueness or comparison semantics change.
|
||||
type Sort struct {
|
||||
ID string
|
||||
Columns []Column
|
||||
}
|
||||
|
||||
// Direction selects which side of an optional anchor to fetch.
|
||||
type Direction uint8
|
||||
|
||||
const (
|
||||
// Forward maps to first/after and is the default direction.
|
||||
Forward Direction = iota
|
||||
// Backward maps to last/before; returned items retain canonical sort order.
|
||||
Backward
|
||||
)
|
||||
|
||||
// Request carries a transport-independent page request. Zero Size uses the
|
||||
// configured default; excessive sizes clamp to Maximum. Empty Cursor has no anchor.
|
||||
type Request struct {
|
||||
Direction Direction
|
||||
Size int
|
||||
Cursor string
|
||||
}
|
||||
|
||||
// Limits configures page sizes; require 1 <= Default <= Maximum < max int.
|
||||
type Limits struct {
|
||||
Default int
|
||||
Maximum int
|
||||
}
|
||||
|
||||
// KeyError identifies a programmer error in the row key's arity.
|
||||
type KeyError struct {
|
||||
Expected int
|
||||
Actual int
|
||||
}
|
||||
|
||||
// Error implements error with the expected and actual key lengths.
|
||||
func (e *KeyError) Error() string {
|
||||
return fmt.Sprintf("page: key length %d, expected %d", e.Actual, e.Expected)
|
||||
}
|
||||
|
||||
// Page contains canonical-order items and cursors for their first and last rows.
|
||||
// Empty pages have empty cursors. Overflow determines the fetch-direction flag;
|
||||
// anchor presence determines the opposite flag, even for empty/stale results.
|
||||
// The opposite flag does not prove another row exists.
|
||||
type Page[T any] struct {
|
||||
Items []T
|
||||
StartCursor string
|
||||
EndCursor string
|
||||
HasNext bool
|
||||
HasPrevious bool
|
||||
}
|
||||
|
||||
// Prepared holds validated pagination and the typed row key. Construct with Prepare.
|
||||
// Treat Sort expressions and the key's captured state as immutable while in use.
|
||||
type Prepared[T any] struct {
|
||||
sort Sort
|
||||
key func(T) []any
|
||||
direction Direction
|
||||
size int
|
||||
anchor []any
|
||||
}
|
||||
|
||||
// Prepare validates a request before a Bob mod can be applied. key must return
|
||||
// exactly one supported concrete value per Sort column, in the same order, using
|
||||
// actual stored/scanned values (including database timestamp precision).
|
||||
func Prepare[T any](sort Sort, key func(T) []any, request Request, limits Limits) (*Prepared[T], error) {
|
||||
if sort.ID == "" || !utf8.ValidString(sort.ID) || len(sort.Columns) == 0 {
|
||||
return nil, fmt.Errorf("page: sort needs a valid ID and columns")
|
||||
}
|
||||
for i, c := range sort.Columns {
|
||||
if c.Expression == nil || c.Direction > Desc {
|
||||
return nil, fmt.Errorf("page: invalid sort column %d", i)
|
||||
}
|
||||
}
|
||||
if !sort.Columns[len(sort.Columns)-1].Unique {
|
||||
return nil, fmt.Errorf("page: final sort column must be unique")
|
||||
}
|
||||
if key == nil {
|
||||
return nil, fmt.Errorf("page: nil key function")
|
||||
}
|
||||
if limits.Default < 1 || limits.Maximum < limits.Default || limits.Maximum == int(^uint(0)>>1) {
|
||||
return nil, fmt.Errorf("page: invalid limits")
|
||||
}
|
||||
if request.Direction > Backward || request.Size < 0 {
|
||||
return nil, fmt.Errorf("page: invalid request direction or size")
|
||||
}
|
||||
size := request.Size
|
||||
if size == 0 {
|
||||
size = limits.Default
|
||||
}
|
||||
size = min(size, limits.Maximum)
|
||||
sort.Columns = slices.Clone(sort.Columns)
|
||||
p := &Prepared[T]{sort: sort, key: key, direction: request.Direction, size: size}
|
||||
if request.Cursor != "" {
|
||||
var err error
|
||||
p.anchor, err = decodeCursor(request.Cursor, sort)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// Page validates all fetched rows, including overflow, and assembles a page.
|
||||
// Pass rows in query order, with at most size+1 rows. It copies retained items;
|
||||
// backward pagination never reverses the caller's slice. Errors return no page.
|
||||
func (p *Prepared[T]) Page(rows []T) (Page[T], error) {
|
||||
var zero Page[T]
|
||||
if len(rows) > p.size+1 {
|
||||
return zero, fmt.Errorf("page: too many fetched rows")
|
||||
}
|
||||
cursors := make([]string, len(rows))
|
||||
for i, row := range rows {
|
||||
key := p.key(row)
|
||||
if len(key) != len(p.sort.Columns) {
|
||||
return zero, &KeyError{Expected: len(p.sort.Columns), Actual: len(key)}
|
||||
}
|
||||
var err error
|
||||
cursors[i], err = encodeCursor(p.sort.ID, key)
|
||||
if err != nil {
|
||||
return zero, err
|
||||
}
|
||||
}
|
||||
overflow := len(rows) > p.size
|
||||
n := min(len(rows), p.size)
|
||||
result := Page[T]{Items: slices.Clone(rows[:n])}
|
||||
if n > 0 {
|
||||
result.StartCursor, result.EndCursor = cursors[0], cursors[n-1]
|
||||
}
|
||||
if p.direction == Backward {
|
||||
slices.Reverse(result.Items)
|
||||
result.StartCursor, result.EndCursor = result.EndCursor, result.StartCursor
|
||||
result.HasPrevious, result.HasNext = overflow, len(p.anchor) > 0
|
||||
} else {
|
||||
result.HasNext, result.HasPrevious = overflow, len(p.anchor) > 0
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package page
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/stephenafamo/bob"
|
||||
"github.com/stephenafamo/bob/clause"
|
||||
"github.com/stephenafamo/bob/expr"
|
||||
"github.com/stephenafamo/scan"
|
||||
)
|
||||
|
||||
// Mod applies validated keyset WHERE, ORDER BY and integer LIMIT size+1 to native
|
||||
// PostgreSQL, SQLite or MySQL selects. Apply once to a query with no existing
|
||||
// ORDER BY, LIMIT, OFFSET or FETCH; do not override pagination afterward.
|
||||
// Callers may compose additional WHERE filters before or after this mod.
|
||||
func Mod[Q interface {
|
||||
AppendWhere(...any)
|
||||
AppendOrder(bob.Expression)
|
||||
SetLimit(any)
|
||||
}, T any](p *Prepared[T]) bob.Mod[Q] {
|
||||
return bob.ModFunc[Q](func(q Q) {
|
||||
var alternatives []bob.Expression
|
||||
for i, c := range p.sort.Columns {
|
||||
desc := c.Direction == Desc
|
||||
if p.direction == Backward {
|
||||
desc = !desc
|
||||
}
|
||||
direction, operator := "ASC", ">"
|
||||
if desc {
|
||||
direction, operator = "DESC", "<"
|
||||
}
|
||||
q.AppendOrder(clause.OrderDef{Expression: c.Expression, Direction: direction})
|
||||
if len(p.anchor) == 0 {
|
||||
continue
|
||||
}
|
||||
terms := make([]bob.Expression, 0, i+1)
|
||||
for j := 0; j < i; j++ {
|
||||
terms = append(terms, expr.OP("=", p.sort.Columns[j].Expression, expr.Arg(p.anchor[j])))
|
||||
}
|
||||
terms = append(terms, expr.OP(operator, c.Expression, expr.Arg(p.anchor[i])))
|
||||
alternatives = append(alternatives, grouped(terms, " AND "))
|
||||
}
|
||||
// authz: group the seek disjunction so caller filters constrain every branch.
|
||||
if len(alternatives) > 0 {
|
||||
q.AppendWhere(grouped(alternatives, " OR "))
|
||||
}
|
||||
q.SetLimit(p.size + 1)
|
||||
})
|
||||
}
|
||||
|
||||
func grouped(expressions []bob.Expression, separator string) bob.Expression {
|
||||
return expr.Join{Exprs: []bob.Expression{
|
||||
expr.Raw("("), expr.Join{Exprs: expressions, Sep: separator}, expr.Raw(")"),
|
||||
}, Sep: expr.NoSep}
|
||||
}
|
||||
|
||||
// Count executes a separate caller-built, authorized count query returning one
|
||||
// int64 column. It never rewrites a paginated query or infers grouped/distinct
|
||||
// semantics. Nothing runs unless called.
|
||||
func Count(ctx context.Context, executor bob.Executor, query bob.Query) (int64, error) {
|
||||
// authz: callers supply the same authorized relation as their item query.
|
||||
count, err := bob.One(ctx, executor, query, scan.SingleColumnMapper[int64])
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("page: count: %w", err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package page_test
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"os"
|
||||
"reflect"
|
||||
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
"github.com/stephenafamo/bob"
|
||||
"github.com/stephenafamo/bob/dialect/mysql"
|
||||
md "github.com/stephenafamo/bob/dialect/mysql/dialect"
|
||||
mm "github.com/stephenafamo/bob/dialect/mysql/sm"
|
||||
"github.com/stephenafamo/bob/dialect/psql"
|
||||
pd "github.com/stephenafamo/bob/dialect/psql/dialect"
|
||||
pm "github.com/stephenafamo/bob/dialect/psql/sm"
|
||||
"github.com/stephenafamo/bob/dialect/sqlite"
|
||||
sd "github.com/stephenafamo/bob/dialect/sqlite/dialect"
|
||||
"github.com/stephenafamo/bob/dialect/sqlite/sm"
|
||||
"github.com/stephenafamo/bob/expr"
|
||||
"github.com/stephenafamo/scan"
|
||||
"go.auvem.com/corestack/page"
|
||||
)
|
||||
|
||||
type item struct {
|
||||
ID int64
|
||||
Score int64
|
||||
}
|
||||
|
||||
func itemSort() page.Sort {
|
||||
return page.Sort{ID: "items/score/v1", Columns: []page.Column{
|
||||
{Expression: expr.Quote("score"), Direction: page.Desc},
|
||||
{Expression: expr.Quote("id"), Direction: page.Asc, Unique: true},
|
||||
}}
|
||||
}
|
||||
|
||||
func itemKey(i item) []any { return []any{i.Score, i.ID} }
|
||||
|
||||
func TestSQLiteFlow(t *testing.T) {
|
||||
db, err := bob.Open("sqlite3", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
databaseFlow(t, db, false)
|
||||
}
|
||||
|
||||
func TestPostgresFlow(t *testing.T) {
|
||||
dsn := os.Getenv("PAGE_POSTGRES_DSN")
|
||||
if dsn == "" {
|
||||
if os.Getenv("PAGE_REQUIRE_POSTGRES") != "" {
|
||||
t.Fatal("PAGE_POSTGRES_DSN is required")
|
||||
}
|
||||
t.Skip("set PAGE_POSTGRES_DSN to run PostgreSQL integration")
|
||||
}
|
||||
db, err := bob.Open("postgres", dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
databaseFlow(t, db, true)
|
||||
}
|
||||
|
||||
func databaseFlow(t *testing.T, db bob.DB, postgres bool) {
|
||||
t.Helper()
|
||||
db.SetMaxOpenConns(1)
|
||||
for _, statement := range []string{
|
||||
"CREATE TEMP TABLE items (id BIGINT PRIMARY KEY, score BIGINT NOT NULL, tenant BIGINT NOT NULL)",
|
||||
"INSERT INTO items VALUES (1,30,1),(2,30,1),(3,20,1),(4,10,1),(5,5,2)",
|
||||
} {
|
||||
if _, err := db.ExecContext(t.Context(), statement); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
fetch := func(direction page.Direction, size int, cursor string) page.Page[item] {
|
||||
t.Helper()
|
||||
p, err := page.Prepare(itemSort(), itemKey, page.Request{Direction: direction, Size: size, Cursor: cursor}, page.Limits{Default: 2, Maximum: 3})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var q bob.Query
|
||||
if postgres {
|
||||
query := psql.Select(pm.Columns("id", "score"), pm.From("items"), pm.Where(psql.Quote("tenant").EQ(psql.Arg(int64(1)))))
|
||||
query.Apply(page.Mod[*pd.SelectQuery](p))
|
||||
query.Apply(pm.Where(psql.Quote("score").GT(psql.Arg(int64(0)))))
|
||||
q = query
|
||||
} else {
|
||||
query := sqlite.Select(sm.Columns("id", "score"), sm.From("items"), sm.Where(sqlite.Quote("tenant").EQ(sqlite.Arg(int64(1)))))
|
||||
query.Apply(page.Mod[*sd.SelectQuery](p))
|
||||
query.Apply(sm.Where(sqlite.Quote("score").GT(sqlite.Arg(int64(0)))))
|
||||
q = query
|
||||
}
|
||||
rows, err := bob.All(t.Context(), db, q, scan.StructMapper[item]())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := append([]item(nil), rows...)
|
||||
result, err := p.Page(rows)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(before, rows) {
|
||||
t.Fatal("Page mutated fetched slice")
|
||||
}
|
||||
return result
|
||||
}
|
||||
check := func(got page.Page[item], ids []int64, next, previous bool) {
|
||||
t.Helper()
|
||||
actual := make([]int64, len(got.Items))
|
||||
for i, row := range got.Items {
|
||||
actual[i] = row.ID
|
||||
}
|
||||
if !reflect.DeepEqual(actual, ids) || got.HasNext != next || got.HasPrevious != previous {
|
||||
t.Fatalf("got IDs %v next=%v previous=%v; want %v %v %v", actual, got.HasNext, got.HasPrevious, ids, next, previous)
|
||||
}
|
||||
if len(ids) == 0 && (got.StartCursor != "" || got.EndCursor != "") {
|
||||
t.Fatal("empty page has cursors")
|
||||
}
|
||||
if len(ids) > 0 && (got.StartCursor == "" || got.EndCursor == "") {
|
||||
t.Fatal("missing cursors")
|
||||
}
|
||||
}
|
||||
first := fetch(page.Forward, 2, "")
|
||||
check(first, []int64{1, 2}, true, false)
|
||||
second := fetch(page.Forward, 2, first.EndCursor)
|
||||
check(second, []int64{3, 4}, false, true)
|
||||
check(fetch(page.Forward, 2, second.EndCursor), []int64{}, false, true)
|
||||
last := fetch(page.Backward, 2, "")
|
||||
check(last, []int64{3, 4}, false, true)
|
||||
previous := fetch(page.Backward, 2, last.StartCursor)
|
||||
check(previous, []int64{1, 2}, true, false)
|
||||
check(fetch(page.Backward, 2, previous.StartCursor), []int64{}, true, false)
|
||||
check(fetch(page.Forward, 0, ""), []int64{1, 2}, true, false)
|
||||
check(fetch(page.Forward, 1000, ""), []int64{1, 2, 3}, true, false)
|
||||
// A cursor describes a value boundary even after its row is deleted.
|
||||
if _, err := db.ExecContext(t.Context(), "DELETE FROM items WHERE id=2"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
check(fetch(page.Forward, 2, first.EndCursor), []int64{3, 4}, false, true)
|
||||
var countQuery, emptyQuery bob.Query
|
||||
if postgres {
|
||||
countQuery = psql.Select(pm.Columns("count(*)"), pm.From("items"), pm.Where(psql.Quote("tenant").EQ(psql.Arg(1))))
|
||||
emptyQuery = psql.Select(pm.Columns("id"), pm.From("items"), pm.Where(psql.Quote("id").LT(psql.Arg(0))))
|
||||
} else {
|
||||
countQuery = sqlite.Select(sm.Columns("count(*)"), sm.From("items"), sm.Where(sqlite.Quote("tenant").EQ(sqlite.Arg(1))))
|
||||
emptyQuery = sqlite.Select(sm.Columns("id"), sm.From("items"), sm.Where(sqlite.Quote("id").LT(sqlite.Arg(0))))
|
||||
}
|
||||
count, err := page.Count(t.Context(), db, countQuery)
|
||||
if err != nil || count != 3 {
|
||||
t.Fatalf("count %d: %v", count, err)
|
||||
}
|
||||
if _, err := page.Count(t.Context(), db, emptyQuery); !errors.Is(err, sql.ErrNoRows) {
|
||||
t.Fatalf("count error lost cause: %v", err)
|
||||
}
|
||||
_, err = page.Prepare(itemSort(), itemKey, page.Request{Cursor: "not!base64"}, page.Limits{Default: 2, Maximum: 3})
|
||||
var cursorError *page.CursorError
|
||||
if !errors.As(err, &cursorError) {
|
||||
t.Fatalf("invalid cursor: %v", err)
|
||||
}
|
||||
bad, err := page.Prepare(itemSort(), func(item) []any { return []any{int64(1)} }, page.Request{}, page.Limits{Default: 2, Maximum: 3})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result, err := bad.Page([]item{{ID: 1}})
|
||||
var keyError *page.KeyError
|
||||
if !errors.As(err, &keyError) || !reflect.DeepEqual(result, page.Page[item]{}) {
|
||||
t.Fatalf("key failure: %+v %v", result, err)
|
||||
}
|
||||
if _, err := db.ExecContext(t.Context(), "DELETE FROM items"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
check(fetch(page.Forward, 2, ""), []int64{}, false, false)
|
||||
check(fetch(page.Backward, 2, ""), []int64{}, false, false)
|
||||
check(fetch(page.Forward, 2, first.EndCursor), []int64{}, false, true)
|
||||
check(fetch(page.Backward, 2, last.StartCursor), []int64{}, true, false)
|
||||
}
|
||||
|
||||
func TestMySQLModComposition(t *testing.T) {
|
||||
p, err := page.Prepare(itemSort(), itemKey, page.Request{}, page.Limits{Default: 2, Maximum: 3})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q := mysql.Select(mm.Columns("id", "score"), mm.From("items"), page.Mod[*md.SelectQuery](p))
|
||||
q.Apply(mm.Where(mysql.Quote("tenant").EQ(mysql.Arg(int64(1)))))
|
||||
// Compile-time composition only; MySQL runtime behavior needs a database test.
|
||||
}
|
||||
Reference in new issue
Block a user