diff --git a/cmd/corestack-goose/internal/authoring/authoring.go b/cmd/corestack-goose/internal/authoring/authoring.go new file mode 100644 index 0000000..5c834e7 --- /dev/null +++ b/cmd/corestack-goose/internal/authoring/authoring.go @@ -0,0 +1,424 @@ +// Package authoring owns the standalone tool's bounded migration source format. +// It requires one writer and a complete source backup; it never accesses a DB. +package authoring + +import ( + "bytes" + "errors" + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "io" + "math" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "text/template" + "time" + + "github.com/pressly/goose/v3" +) + +const marker = "// Code generated by corestack-goose. DO NOT EDIT.\n" +const goTemplate = `package migrations +import ( + "context" + "database/sql" + "github.com/pressly/goose/v3" +) +func migration{{.CamelName}}(version int64) *goose.Migration { + return goose.NewGoMigration(version, &goose.GoFunc{RunTx: up{{.CamelName}}}, &goose.GoFunc{RunTx: down{{.CamelName}}}) +} +func up{{.CamelName}}(ctx context.Context, tx *sql.Tx) error { + return nil +} +func down{{.CamelName}}(ctx context.Context, tx *sql.Tx) error { + return nil +} +` + +type migration struct { + name, factory string + version int64 + data []byte +} + +type change struct { + source, dest string + data []byte + replace bool +} + +type sourceSet struct { + dir string + migrations []migration + register []byte +} + +// Create stages Goose's supported templates, then publishes a preflighted set. +// Go names use lowercase ASCII words separated by underscores. All versions and +// identifiers must be unique across the directory; existing bodies are preserved. +func Create(dir, name, kind string) error { + if !validName(name) || (kind != "sql" && kind != "go") { + return errors.New("create: expected lowercase_words name and sql|go") + } + set, err := scan(dir) + if err != nil { + return err + } + stage, err := os.MkdirTemp("", "corestack-goose-*") + if err != nil { + return err + } + defer os.RemoveAll(stage) + var tmpl *template.Template + if kind == "go" { + tmpl, err = template.New("migration").Parse(goTemplate) + if err != nil { + return err + } + } + if err := goose.CreateWithTemplate(nil, stage, tmpl, name, kind); err != nil { + return fmt.Errorf("stage creation: %w", err) + } + entries, err := os.ReadDir(stage) + if err != nil { + return err + } + if len(entries) != 1 { + return errors.New("stage creation: expected one migration") + } + filename := entries[0].Name() + data, err := os.ReadFile(filepath.Join(stage, filename)) + if err != nil { + return err + } + if kind == "go" { + filename = strings.TrimSuffix(filename, ".go") + "_migration.go" + data, err = format.Source(data) + if err != nil { + return err + } + } + item, err := parseMigration(filename, data) + if err != nil { + return err + } + items := append(slices.Clone(set.migrations), item) + if err := validate(items); err != nil { + return err + } + register, err := aggregate(items) + if err != nil { + return err + } + changes := []change{{dest: filename, data: data}} + changes = appendRegister(changes, set, register) + return publish(set, changes, writeExclusive) +} + +// Fix preserves sequential IDs and allocates timestamp IDs after their maximum. +// It never rewrites migration bodies or database history. Commit or separately +// back up all sources first; on failure restore that backup before retrying. +func Fix(dir string) error { + set, err := scan(dir) + if err != nil { + return err + } + items := slices.Clone(set.migrations) + var highest int64 + for _, item := range items { + if !timestamp(item.version) && item.version > highest { + highest = item.version + } + } + var changes []change + for i := range items { + if !timestamp(items[i].version) { + continue + } + if highest == math.MaxInt64 { + return errors.New("fix: sequential version overflow") + } + highest++ + old := items[i].name + items[i].version = highest + items[i].name = fmt.Sprintf("%05d_%s", highest, strings.SplitN(old, "_", 2)[1]) + changes = append(changes, change{source: old, dest: items[i].name, data: items[i].data}) + } + if err := validate(items); err != nil { + return err + } + register, err := aggregate(items) + if err != nil { + return err + } + changes = appendRegister(changes, set, register) + return publish(set, changes, writeExclusive) +} + +func timestamp(version int64) bool { + t, err := time.Parse("20060102150405", strconv.FormatInt(version, 10)) + return err == nil && t.Unix() > 0 +} + +func validName(name string) bool { + for _, word := range strings.Split(name, "_") { + if word == "" { + return false + } + for _, r := range word { + if r < 'a' || r > 'z' { + return false + } + } + } + return true +} + +func parseMigration(name string, data []byte) (migration, error) { + item := migration{name: name, data: data} + ext := filepath.Ext(name) + parts := strings.SplitN(strings.TrimSuffix(name, ext), "_", 2) + if len(parts) != 2 { + return item, fmt.Errorf("unsupported migration filename %s", name) + } + raw := parts[0] + if strings.IndexFunc(raw, func(r rune) bool { return r < '0' || r > '9' }) >= 0 { + return item, fmt.Errorf("invalid version in %s", name) + } + v, err := strconv.ParseInt(raw, 10, 64) + if err != nil || v <= 0 { + return item, fmt.Errorf("invalid positive int64 version in %s", name) + } + item.version = v + if ext == ".sql" { + return item, nil + } + logical, ok := strings.CutSuffix(parts[1], "_migration") + if !ok || !validName(logical) { + return item, fmt.Errorf("unsupported Go filename %s: use __migration.go", name) + } + var camel strings.Builder + for _, word := range strings.Split(logical, "_") { + camel.WriteString(strings.ToUpper(word[:1]) + word[1:]) + } + item.factory = "migration" + camel.String() + file, err := parser.ParseFile(token.NewFileSet(), name, data, 0) + if err != nil { + return item, fmt.Errorf("parse %s: %w", name, err) + } + if file.Name.Name != "migrations" { + return item, fmt.Errorf("%s: expected package migrations", name) + } + count := 0 + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Name.Name != item.factory { + continue + } + var signature bytes.Buffer + if err := format.Node(&signature, token.NewFileSet(), fn.Type); err != nil { + return item, err + } + if fn.Recv != nil || signature.String() != "func(version int64) *goose.Migration" || fn.Body == nil { + return item, fmt.Errorf("%s: expected %s(version int64) *goose.Migration factory", name, item.factory) + } + count++ + } + if count != 1 { + return item, fmt.Errorf("%s: expected one %s factory", name, item.factory) + } + return item, nil +} + +func validate(items []migration) error { + slices.SortFunc(items, func(a, b migration) int { + if a.version < b.version { + return -1 + } + if a.version > b.version { + return 1 + } + return 0 + }) + factories := make(map[string]bool) + for i, item := range items { + if i > 0 { + previous := items[i-1] + if previous.version == item.version { + return fmt.Errorf("duplicate version: %s and %s", previous.name, item.name) + } + if previous.name >= item.name { + return fmt.Errorf("lexical/numeric order disagreement: %s and %s; use consistent padding", previous.name, item.name) + } + } + if item.factory != "" { + if factories[item.factory] { + return fmt.Errorf("duplicate factory %s", item.factory) + } + factories[item.factory] = true + } + } + return nil +} + +func aggregate(items []migration) ([]byte, error) { + var out strings.Builder + out.WriteString(marker + "\n// Package migrations contains application-owned SQL and Go migrations.\n// corestack-goose owns this aggregate; registrations are provider-local.\npackage migrations\n\nimport \"github.com/pressly/goose/v3\"\n\n// GoMigrations returns fresh provider-local registrations on every call.\nfunc GoMigrations() []*goose.Migration {\nreturn []*goose.Migration{\n") + for _, item := range items { + if item.factory != "" { + fmt.Fprintf(&out, "%s(%d),\n", item.factory, item.version) + } + } + out.WriteString("}\n}\n") + return format.Source([]byte(out.String())) +} + +func scan(dir string) (sourceSet, error) { + set := sourceSet{dir: dir} + absolute, err := filepath.Abs(dir) + if err != nil { + return set, err + } + resolved, err := filepath.EvalSymlinks(absolute) + if err != nil { + return set, err + } + if resolved != absolute { + return set, fmt.Errorf("%s: symlink paths are unsupported", dir) + } + entries, err := os.ReadDir(dir) + if err != nil { + return set, err + } + for _, entry := range entries { + name := entry.Name() + if name != "register.go" && filepath.Ext(name) != ".go" && filepath.Ext(name) != ".sql" { + continue + } + info, err := entry.Info() + if err != nil { + return set, err + } + if !info.Mode().IsRegular() { + return set, fmt.Errorf("%s: expected regular file", name) + } + data, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + return set, err + } + if name == "register.go" { + set.register = data + continue + } + item, err := parseMigration(name, data) + if err != nil { + return set, err + } + set.migrations = append(set.migrations, item) + } + if err := validate(set.migrations); err != nil { + return set, err + } + expected, err := aggregate(set.migrations) + if err != nil { + return set, err + } + if set.register != nil { + normalized, err := format.Source(set.register) + if err != nil || !bytes.HasPrefix(set.register, []byte(marker)) || !bytes.Equal(normalized, expected) { + return set, errors.New("register.go: unsupported or drifted aggregate; restore the complete source backup") + } + } else { + for _, item := range set.migrations { + if item.factory != "" { + return set, errors.New("register.go missing: restore the complete source backup") + } + } + } + return set, nil +} + +func appendRegister(changes []change, set sourceSet, data []byte) []change { + if bytes.Equal(data, set.register) { + return changes + } + return append(changes, change{dest: "register.go", data: data, replace: set.register != nil}) +} + +func writeExclusive(path string, data []byte) error { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666) + if err != nil { + return err + } + _, writeErr := io.Copy(file, bytes.NewReader(data)) + return errors.Join(writeErr, file.Close()) +} + +// invariant: publish complete destinations before deleting sources; a failure +// stops the plan and requires manual source restoration, never an automatic retry. +func publish(set sourceSet, changes []change, write func(string, []byte) error) error { + if len(changes) == 0 { + return nil + } + for _, op := range changes { + if !op.replace { + if _, err := os.Lstat(filepath.Join(set.dir, op.dest)); !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("destination %s exists or cannot be checked: %w", op.dest, errors.Join(err, os.ErrExist)) + } + } + } + stage, err := os.MkdirTemp(set.dir, ".corestack-goose-*") + if err != nil { + return err + } + defer os.RemoveAll(stage) + for i, op := range changes { + if err := writeExclusive(filepath.Join(stage, strconv.Itoa(i)), op.data); err != nil { + return fmt.Errorf("stage %s: %w", op.dest, err) + } + } + var completed []string + for i, op := range changes { + dest := filepath.Join(set.dir, op.dest) + if op.replace { + var current []byte + info, statErr := os.Lstat(dest) + if statErr != nil || !info.Mode().IsRegular() { + err = fmt.Errorf("aggregate is no longer regular: %w", errors.Join(statErr, os.ErrInvalid)) + } else { + current, err = os.ReadFile(dest) + if err == nil && !bytes.Equal(current, set.register) { + err = errors.New("aggregate changed since preflight") + } + if err == nil { + err = os.Rename(filepath.Join(stage, strconv.Itoa(i)), dest) + } + } + } else { + err = write(dest, op.data) + } + if err == nil { + completed = append(completed, op.dest) + if op.source != "" { + err = os.Remove(filepath.Join(set.dir, op.source)) + if err == nil { + completed = append(completed, "removed "+op.source) + } + } + } + if err != nil { + var outstanding []string + for _, pending := range changes[i:] { + outstanding = append(outstanding, fmt.Sprintf("%s -> %s", pending.source, pending.dest)) + } + return fmt.Errorf("authoring stopped: %w; completed %v; inspect current/partial destination %s; outstanding %v; restore complete source backup before retry", err, completed, op.dest, outstanding) + } + } + return nil +} diff --git a/cmd/corestack-goose/internal/authoring/authoring_test.go b/cmd/corestack-goose/internal/authoring/authoring_test.go new file mode 100644 index 0000000..49505af --- /dev/null +++ b/cmd/corestack-goose/internal/authoring/authoring_test.go @@ -0,0 +1,313 @@ +package authoring + +import ( + "bytes" + "errors" + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func directory(t *testing.T) string { + t.Helper() + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return dir +} +func put(t *testing.T, dir, name string, data []byte) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), data, 0o600); err != nil { + t.Fatal(err) + } +} +func snapshot(t *testing.T, dir string) map[string]string { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + result := map[string]string{} + for _, entry := range entries { + if entry.IsDir() { + result[entry.Name()] = "" + continue + } + data, err := os.ReadFile(filepath.Join(dir, entry.Name())) + if err != nil { + t.Fatal(err) + } + result[entry.Name()] = string(data) + } + return result +} +func seedGo(t *testing.T, dir, name string, version int64) { + t.Helper() + if err := Create(dir, name, "go"); err != nil { + t.Fatal(err) + } + set, err := scan(dir) + if err != nil { + t.Fatal(err) + } + item := set.migrations[len(set.migrations)-1] + old := item.name + item.name = fmt.Sprintf("%05d_%s", version, strings.SplitN(old, "_", 2)[1]) + item.version = version + if err := os.Rename(filepath.Join(dir, old), filepath.Join(dir, item.name)); err != nil { + t.Fatal(err) + } + set.migrations[len(set.migrations)-1] = item + data, err := aggregate(set.migrations) + if err != nil { + t.Fatal(err) + } + put(t, dir, "register.go", data) +} +func TestCreateFixAndPreserveEdits(t *testing.T) { + dir := directory(t) + seedGo(t, dir, "test", 20261028093000) + name := "20261028093000_test_migration.go" + before := snapshot(t, dir)[name] + edited := strings.Replace(before, "return nil", "// application body\n return tx.QueryRowContext(ctx, \"SELECT 1\").Err()", 1) + put(t, dir, name, []byte(edited)) + put(t, dir, "00007_existing.sql", []byte("-- +goose Up\nSELECT 1;\n-- +goose Down\nSELECT 1;\n")) + put(t, dir, "20261028103000_next.sql", []byte("-- +goose Up\nSELECT 2;\n-- +goose Down\nSELECT 2;\n")) + if err := Fix(dir); err != nil { + t.Fatal(err) + } + after := snapshot(t, dir) + if after["00008_test_migration.go"] != edited || after["00009_next.sql"] == "" || !strings.Contains(after["register.go"], "migrationTest(8)") { + t.Fatal(after) + } + if err := Fix(dir); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(after, snapshot(t, dir)) { + t.Fatal("repeated fix changed set") + } + if err := Create(dir, "new_sql", "sql"); err != nil { + t.Fatal(err) + } + if err := Fix(dir); err != nil { + t.Fatal(err) + } + if snapshot(t, dir)["00010_new_sql.sql"] == "" { + t.Fatal("creation failed to preserve sequence") + } +} +func TestPreflightRejections(t *testing.T) { + for _, tc := range []struct { + name string + prepare func(*testing.T, string) + want string + }{ + {"duplicate", func(t *testing.T, d string) { put(t, d, "00001_one.sql", nil); put(t, d, "1_two.sql", nil) }, "duplicate version"}, + {"invalid", func(t *testing.T, d string) { put(t, d, "-1_bad.sql", nil) }, "invalid version"}, + {"zero", func(t *testing.T, d string) { put(t, d, "00000_bad.sql", nil) }, "positive int64"}, + {"overflow", func(t *testing.T, d string) { put(t, d, "9223372036854775808_bad.sql", nil) }, "positive int64"}, + {"order", func(t *testing.T, d string) { put(t, d, "2_two.sql", nil); put(t, d, "10_ten.sql", nil) }, "order disagreement"}, + {"output-order", func(t *testing.T, d string) { put(t, d, "1_one.sql", nil); put(t, d, "20261028093000_two.sql", nil) }, "order disagreement"}, + {"factory", func(t *testing.T, d string) { + seedGo(t, d, "example", 1) + p := filepath.Join(d, "00001_example_migration.go") + b, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + put(t, d, filepath.Base(p), bytes.ReplaceAll(b, []byte("version int64"), []byte("version int"))) + }, "expected migrationExample"}, + {"package", func(t *testing.T, d string) { + seedGo(t, d, "example", 1) + b := snapshot(t, d)["00001_example_migration.go"] + put(t, d, "00001_example_migration.go", []byte(strings.Replace(b, "package migrations", "package other", 1))) + }, "package migrations"}, + {"stock", func(t *testing.T, d string) { put(t, d, "00001_example.go", []byte("package migrations")) }, "unsupported Go filename"}, + {"missing-aggregate", func(t *testing.T, d string) { + seedGo(t, d, "example", 1) + if err := os.Remove(filepath.Join(d, "register.go")); err != nil { + t.Fatal(err) + } + }, "register.go missing"}, + {"drift", func(t *testing.T, d string) { + seedGo(t, d, "example", 1) + b := snapshot(t, d)["register.go"] + put(t, d, "register.go", []byte(strings.Replace(b, "migrationExample(1)", "migrationExample(2)", 1))) + }, "drifted aggregate"}, + {"handwritten", func(t *testing.T, d string) { put(t, d, "register.go", []byte("package migrations")) }, "drifted aggregate"}, + {"destination-dir", func(t *testing.T, d string) { + put(t, d, "20261028093000_one.sql", nil) + if err := os.Mkdir(filepath.Join(d, "00001_one.sql"), 0o700); err != nil { + t.Fatal(err) + } + }, "regular file"}, + {"sequential-overflow", func(t *testing.T, d string) { + put(t, d, "0000020261028093000_one.sql", nil) + put(t, d, "9223372036854775807_last.sql", nil) + }, "sequential version overflow"}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := directory(t) + tc.prepare(t, dir) + before := snapshot(t, dir) + // Directories are checked separately; every file must survive preflight unchanged. + err := Fix(dir) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("wanted %s: %v", tc.want, err) + } + if !reflect.DeepEqual(before, snapshot(t, dir)) { + t.Fatal("preflight failure changed files") + } + }) + } +} +func TestCreateValidationAndCollisions(t *testing.T) { + for _, name := range []string{"", "../bad", "Test", "bad__name", "bad2", "bad-name"} { + if err := Create(directory(t), name, "go"); err == nil { + t.Fatalf("accepted %q", name) + } + } + if err := Create(directory(t), "valid", "other"); err == nil { + t.Fatal("accepted unsupported type") + } + dir := directory(t) + seedGo(t, dir, "example", 1) + before := snapshot(t, dir) + if err := Create(dir, "example", "go"); err == nil || !strings.Contains(err.Error(), "duplicate factory") { + t.Fatal(err) + } + if !reflect.DeepEqual(before, snapshot(t, dir)) { + t.Fatal("rejected create changed sources") + } +} +func TestSymlinksRejected(t *testing.T) { + dir := directory(t) + link := filepath.Join(directory(t), "link") + if err := os.Symlink(dir, link); err != nil { + t.Fatal(err) + } + if err := Fix(link); err == nil || !strings.Contains(err.Error(), "symlink") { + t.Fatal(err) + } + source := filepath.Join(directory(t), "source") + if err := os.WriteFile(source, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(source, filepath.Join(dir, "00001_one.sql")); err != nil { + t.Fatal(err) + } + if err := Fix(dir); err == nil || !strings.Contains(err.Error(), "regular file") { + t.Fatal(err) + } +} +func TestTimestampClassification(t *testing.T) { + for _, tc := range []struct { + v int64 + want bool + }{{1, false}, {19700101000000, false}, {19700101000001, true}, {19691231235959, false}, {20260230093000, false}, {20261028093000, true}} { + if got := timestamp(tc.v); got != tc.want { + t.Fatalf("%d timestamp=%v", tc.v, got) + } + } +} +func TestPartialPublicationAndManualRecovery(t *testing.T) { + dir := directory(t) + seedGo(t, dir, "example", 20261028093000) + before := snapshot(t, dir) + set, err := scan(dir) + if err != nil { + t.Fatal(err) + } + item := set.migrations[0] + item.name = "00001_example_migration.go" + item.version = 1 + aggregateData, err := aggregate([]migration{item}) + if err != nil { + t.Fatal(err) + } + changes := []change{{source: set.migrations[0].name, dest: item.name, data: item.data}, {dest: "00002_next.sql", data: []byte("example")}, {dest: "register.go", replace: true, data: aggregateData}} + failure := errors.New("injected disk failure") + err = publish(set, changes, func(path string, data []byte) error { + if strings.HasSuffix(path, "00002_next.sql") { + if err := writeExclusive(path, []byte("partial")); err != nil { + t.Fatal(err) + } + return failure + } + return writeExclusive(path, data) + }) + if !errors.Is(err, failure) || !strings.Contains(err.Error(), "completed [00001_example_migration.go removed 20261028093000_example_migration.go]") || !strings.Contains(err.Error(), "outstanding") || !strings.Contains(err.Error(), "restore complete source backup") { + t.Fatal(err) + } + if snapshot(t, dir)["00002_next.sql"] != "partial" { + t.Fatal("partial destination not reported/preserved") + } + if err := Fix(dir); err == nil { + t.Fatal("inconsistent interrupted set accepted") + } + // Manual recovery restores all paths, including untracked sources and aggregate. + for name := range snapshot(t, dir) { + if err := os.Remove(filepath.Join(dir, name)); err != nil { + t.Fatal(err) + } + } + for name, data := range before { + put(t, dir, name, []byte(data)) + } + if err := Fix(dir); err != nil { + t.Fatal(err) + } + if snapshot(t, dir)["00001_example_migration.go"] == "" { + t.Fatal("recovered fix failed") + } +} +func TestExclusiveDestinationAndAggregateRecheck(t *testing.T) { + dir := directory(t) + if err := Fix(dir); err != nil { + t.Fatal(err) + } + set, err := scan(dir) + if err != nil { + t.Fatal(err) + } + put(t, dir, "00001_collision.sql", []byte("existing")) + err = publish(set, []change{{dest: "00001_collision.sql", data: []byte("replace")}}, writeExclusive) + if err == nil || snapshot(t, dir)["00001_collision.sql"] != "existing" { + t.Fatal("overwrote destination", err) + } + err = writeExclusive(filepath.Join(dir, "00001_collision.sql"), []byte("replace")) + if !errors.Is(err, os.ErrExist) { + t.Fatal(err) + } + err = publish(set, []change{{dest: "00002_ok.sql"}, {dest: "register.go", replace: true, data: []byte("new")}}, func(path string, data []byte) error { + put(t, dir, "register.go", []byte("editor changed")) + return writeExclusive(path, data) + }) + if err == nil || !strings.Contains(err.Error(), "aggregate changed since preflight") || snapshot(t, dir)["register.go"] != "editor changed" { + t.Fatal(err) + } +} + +func TestRemovalFailureStopsPublication(t *testing.T) { + dir := directory(t) + source := "20261028093000_one.sql" + if err := os.Mkdir(filepath.Join(dir, source), 0o700); err != nil { + t.Fatal(err) + } + put(t, filepath.Join(dir, source), "block-removal", nil) + err := publish(sourceSet{dir: dir}, []change{{source: source, dest: "00001_one.sql", data: []byte("complete destination")}, {dest: "00002_later.sql"}}, writeExclusive) + if err == nil || !strings.Contains(err.Error(), "completed [00001_one.sql]") || !strings.Contains(err.Error(), "outstanding") { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dir, "00002_later.sql")); !errors.Is(err, os.ErrNotExist) { + t.Fatal("continued after remove failure", err) + } + data, err := os.ReadFile(filepath.Join(dir, "00001_one.sql")) + if err != nil || string(data) != "complete destination" { + t.Fatalf("destination lost: %s %v", data, err) + } +} diff --git a/cmd/corestack-goose/main.go b/cmd/corestack-goose/main.go new file mode 100644 index 0000000..0d693ea --- /dev/null +++ b/cmd/corestack-goose/main.go @@ -0,0 +1,40 @@ +// Command corestack-goose authors provider-local SQL and Go migrations offline. +package main + +import ( + "context" + "fmt" + "os" + + urfavecli "github.com/urfave/cli/v3" + "go.auvem.com/corestack/cmd/corestack-goose/internal/authoring" +) + +func main() { + if err := command().Run(context.Background(), os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func command() *urfavecli.Command { + root := &urfavecli.Command{Name: "corestack-goose", Usage: "Author migrations before deployment"} + for _, name := range []string{"create", "fix"} { + root.Commands = append(root.Commands, &urfavecli.Command{Name: name, + Flags: []urfavecli.Flag{&urfavecli.StringFlag{Name: "dir", Required: true}}, + Action: func(_ context.Context, cmd *urfavecli.Command) error { + if name == "create" { + if cmd.NArg() != 2 { + return fmt.Errorf("usage: create --dir sql|go") + } + return authoring.Create(cmd.String("dir"), cmd.Args().Get(0), cmd.Args().Get(1)) + } + if cmd.NArg() != 0 { + return fmt.Errorf("usage: fix --dir ") + } + return authoring.Fix(cmd.String("dir")) + }, + }) + } + return root +} diff --git a/cmd/corestack-goose/main_test.go b/cmd/corestack-goose/main_test.go new file mode 100644 index 0000000..caba350 --- /dev/null +++ b/cmd/corestack-goose/main_test.go @@ -0,0 +1,123 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" +) + +func TestGeneratedConsumerAndArtifactBoundary(t *testing.T) { + root, err := filepath.Abs("../..") + if err != nil { + t.Fatal(err) + } + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + execute := func(cwd, name string, args ...string) []byte { + t.Helper() + cmd := exec.CommandContext(t.Context(), name, args...) + cmd.Dir = cwd + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%s %v: %v\n%s", name, args, err, out) + } + return out + } + tool := filepath.Join(dir, "corestack-goose") + execute(root, "go", "build", "-o", tool, "./cmd/corestack-goose") + migrations := filepath.Join(dir, "migrations") + if err := os.Mkdir(migrations, 0o700); err != nil { + t.Fatal(err) + } + osName := "windows" + if runtime.GOOS == "windows" { + osName = "linux" + } + for _, tc := range []struct{ name, kind string }{{"initial", "sql"}, {"test", "go"}, {"create_test", "go"}, {osName, "go"}} { + execute(dir, tool, "create", "--dir", migrations, tc.name, tc.kind) + execute(dir, tool, "fix", "--dir", migrations) + } + execute(dir, tool, "fix", "--dir", migrations) + source, err := os.ReadFile("testdata/consumer/main.go") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "main.go"), source, 0o600); err != nil { + t.Fatal(err) + } + module := fmt.Sprintf("module example.com/consumer\n\ngo 1.27.1\n\nrequire go.auvem.com/corestack v0.0.0\nreplace go.auvem.com/corestack => %s\n", filepath.ToSlash(root)) + if err := os.WriteFile(filepath.Join(dir, "go.mod"), []byte(module), 0o600); err != nil { + t.Fatal(err) + } + binary := filepath.Join(dir, "consumer") + execute(dir, "go", "build", "-mod=mod", "-o", binary, ".") + out := execute(dir, binary, filepath.Join(dir, "consumer.db")) + if !strings.Contains(string(out), "version=4 pending=false versions=1,2,3,4 fresh=true isolated-pending=true") { + t.Fatal(string(out)) + } + // Decode package identities rather than guessing from substring matches in go list. + deps := execute(dir, "go", "list", "-deps", "-json", ".") + decoder := json.NewDecoder(bytes.NewReader(deps)) + adapterFound := false + for decoder.More() { + var dep struct{ ImportPath string } + if err := decoder.Decode(&dep); err != nil { + t.Fatal(err) + } + if dep.ImportPath == "go.auvem.com/corestack/migrate/lifecycle" { + adapterFound = true + } + if dep.ImportPath == "go.auvem.com/corestack/cmd/corestack-goose" || strings.HasPrefix(dep.ImportPath, "go.auvem.com/corestack/cmd/corestack-goose/") { + t.Fatalf("production imports authoring: %s", dep.ImportPath) + } + } + if !adapterFound { + t.Fatal("production fixture did not import runtime adapter") + } + toolSymbols := execute(dir, "go", "tool", "nm", tool) + toolEntries := map[string]bool{} + for _, line := range strings.Split(string(toolSymbols), "\n") { + fields := strings.Fields(line) + if len(fields) >= 3 { + toolEntries[fields[len(fields)-1]] = true + } + } + for _, entry := range []string{"github.com/pressly/goose/v3.CreateWithTemplate", "go.auvem.com/corestack/cmd/corestack-goose/internal/authoring.Create", "go.auvem.com/corestack/cmd/corestack-goose/internal/authoring.Fix"} { + if !toolEntries[entry] { + t.Fatalf("standalone authoring entrypoint missing: %s", entry) + } + } + runtimeFound := false + symbols := execute(dir, "go", "tool", "nm", binary) + for _, line := range strings.Split(string(symbols), "\n") { + fields := strings.Fields(line) + if len(fields) < 3 { + continue + } + symbol := fields[len(fields)-1] + if symbol == "go.auvem.com/corestack/migrate/lifecycle.Commands" { + runtimeFound = true + } + if strings.HasPrefix(symbol, "go.auvem.com/corestack/cmd/corestack-goose/") { + t.Fatalf("authoring symbol linked: %s", symbol) + } + for _, entry := range []string{"Create", "CreateWithTemplate", "Fix", "SetSequential"} { + full := "github.com/pressly/goose/v3." + entry + if symbol == full || strings.HasPrefix(symbol, full+".") { + t.Fatalf("upstream authoring entrypoint linked: %s", symbol) + } + } + } + if !runtimeFound { + t.Fatal("runtime command symbol missing from production executable") + } + t.Log("standalone create/fix; unchanged generated SQL+safe-suffix Go compiled and applied; runtime dependency identities and linker symbols verified") +} diff --git a/cmd/corestack-goose/testdata/consumer/main.go b/cmd/corestack-goose/testdata/consumer/main.go new file mode 100644 index 0000000..d9b88b5 --- /dev/null +++ b/cmd/corestack-goose/testdata/consumer/main.go @@ -0,0 +1,90 @@ +// Command consumer verifies generated registrations and the production boundary. +package main + +import ( + "context" + "database/sql" + "embed" + "fmt" + "io" + "log/slog" + "os" + + "example.com/consumer/migrations" + "github.com/pressly/goose/v3" + urfavecli "github.com/urfave/cli/v3" + "go.auvem.com/corestack/lifecycle" + "go.auvem.com/corestack/lifecycle/cli" + "go.auvem.com/corestack/migrate" + adapter "go.auvem.com/corestack/migrate/lifecycle" + _ "modernc.org/sqlite" +) + +//go:embed migrations +var files embed.FS + +func main() { + ctx := context.Background() + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + var db *sql.DB + dbModule := lifecycle.Module{Name: "database", Setup: func(context.Context) error { var err error; db, err = sql.Open("sqlite", os.Args[1]); return err }, Teardown: func(context.Context) error { return db.Close() }} + resolver := func(context.Context) (*goose.Provider, error) { + return migrate.New(db, goose.DialectSQLite3, files, "migrations", logger, migrations.GoMigrations()...) + } + root := &urfavecli.Command{Name: "consumer", Commands: []*urfavecli.Command{adapter.Commands(lifecycle.Config{Logger: logger}, []lifecycle.Module{dbModule}, resolver)}} + code, err := cli.Run(ctx, root, []string{"consumer", "migrate", "up"}) + must(err) + if code != 0 { + panic(code) + } + db, err = sql.Open("sqlite", os.Args[1]) + must(err) + defer db.Close() + p, err := resolver(ctx) + must(err) + version, err := p.GetDBVersion(ctx) + must(err) + if version != 4 { + panic(version) + } + pending, err := p.HasPending(ctx) + must(err) + if pending { + panic("pending after apply") + } + statuses, err := p.Status(ctx) + must(err) + if len(statuses) != 4 { + panic(len(statuses)) + } + for i, status := range statuses { + if status.Source.Version != int64(i+1) || status.State != goose.StateApplied { + panic(status) + } + } + a, b := migrations.GoMigrations(), migrations.GoMigrations() + if len(a) != 3 || len(b) != 3 { + panic("missing safe-suffix Go migrations") + } + for i := range a { + if a[i] == b[i] || a[i].Version != int64(i+2) || b[i].Version != int64(i+2) { + panic("registration/version isolation") + } + } + isolatedDB, err := sql.Open("sqlite", ":memory:") + must(err) + defer isolatedDB.Close() + isolated, err := migrate.New(isolatedDB, goose.DialectSQLite3, files, "migrations", logger, migrations.GoMigrations()...) + must(err) + pending, err = isolated.HasPending(ctx) + must(err) + if !pending { + panic("provider isolation") + } + fmt.Println("version=4 pending=false versions=1,2,3,4 fresh=true isolated-pending=true") +} +func must(err error) { + if err != nil { + panic(err) + } +} diff --git a/docs/adr/20261028T215512402Z-separate-migration-authoring-tools-from-lifecycle-runtime-adapters.md b/docs/adr/20261028T215512402Z-separate-migration-authoring-tools-from-lifecycle-runtime-adapters.md index 75488af..c2495c1 100644 --- a/docs/adr/20261028T215512402Z-separate-migration-authoring-tools-from-lifecycle-runtime-adapters.md +++ b/docs/adr/20261028T215512402Z-separate-migration-authoring-tools-from-lifecycle-runtime-adapters.md @@ -2,7 +2,7 @@ title: "Separate migration authoring tools from lifecycle runtime adapters" date: 2026-10-28 status: accepted -implementation: not-started +implementation: in-progress summary: Keep startup and native migration commands in a thin lifecycle adapter and source creation and reconciliation in a standalone development tool. tags: - migrations @@ -36,7 +36,7 @@ Preserve the accepted lifecycle ordering, reverse cleanup, caller-owned resource ## Decision Outcome -Choose the third option in the root module, with no new dependencies or lifecycle core changes. The owner accepted this decision, including `up-to` and single-step `down`. Implementation has not started; independent delta review precedes production code. +Choose the third option in the root module, with no new dependencies or lifecycle core changes. The owner accepted this decision, including `up-to` and single-step `down`. Implementation is in progress after independent ADR review; implementation review and publication remain outstanding. Runtime lives in `go.auvem.com/corestack/migrate/lifecycle` (package lifecycle, examples alias migratelifecycle). It may import migrate's dependencies and `lifecycle/cli`; neither migrate nor lifecycle imports it. Authoring lives only in `cmd/corestack-goose` and its internal implementation, never imported by runtime packages or generated consumer code. @@ -97,8 +97,10 @@ There is no atomic multi-file guarantee. Stage first, fully publish each destina * Delivery touches adapter/tool files, migrate authoring guidance/doc comment, focused tests, examples and fixtures; no current callers, publication configuration or dependencies need changes. Assess authoring complexity during implementation; escalate material growth such as arbitrary-source compatibility or automated crash recovery. Consumer docs recommend the standalone flow and link Goose concepts rather than reproduce its manual. * Validate both startup policies (including require blocking later startup), all five leaves against SQLite, lifecycle/context/error paths, help/usage without database setup, and status writer failures. Cover version validation and Goose zero behavior, UpTo targets and single-step Down. * Build a runtime-only production fixture and standalone tool separately. Check production dependencies/binary symbols exclude this project's authoring package and file-writing entry points; Goose's upstream authoring source may remain in the dependency. Build/test both artifacts. -* The decisive flow creates SQL/Go, fixes timestamp IDs, compiles unedited registrations (including test, create_test and a host-incompatible platform name), applies to SQLite, and verifies versions, pending state and provider isolation. Also test collisions, unsupported/drifted aggregates, duplicate/mixed IDs, repeated fix, and injected partial-write failure followed by documented source restoration. These remain delivery gates, not completed checks. +* The decisive flow creates SQL/Go, fixes timestamp IDs, compiles unedited registrations (including test, create_test and a host-incompatible platform name), applies to SQLite, and verifies versions, pending state and provider isolation. Also test collisions, unsupported/drifted aggregates, duplicate/mixed IDs, repeated fix, and injected partial-write failure followed by documented source restoration. These checks pass locally; independent implementation review and publication remain outstanding. ## More Information -Feasibility is source-inspected, not validated end to end. Pinned Goose v3.29.0 [CreateWithTemplate](https://github.com/pressly/goose/blob/v3.29.0/create.go) provides Version/CamelName template values and exclusive creation, but no returned path and possible partial template-error output, motivating staging. [Fix](https://github.com/pressly/goose/blob/v3.29.0/fix.go) uses os.Rename without aggregate maintenance or no-overwrite guarantees; reuse creation, not that reconciliation. [NewGoMigration](https://github.com/pressly/goose/blob/v3.29.0/migration.go) and [provider options](https://github.com/pressly/goose/blob/v3.29.0/provider_options.go) support fresh provider-local registration. [Provider operations](https://github.com/pressly/goose/blob/v3.29.0/provider.go) and [initialization](https://github.com/pressly/goose/blob/v3.29.0/provider_run.go) establish ownership and version-table behavior. Native [argument validation](https://github.com/urfave/cli/blob/v3.14.0/command_run.go) precedes Action. These sources were read locally; integration and artifact exclusion await implementation. +The early real flow built the standalone tool, generated SQL and Go sources, reconciled timestamps, compiled unchanged registrations and applied them to SQLite with fresh-provider isolation. Focused fixtures exercise the production runtime boundary; final checks and independent implementation review remain delivery gates. Pinned Goose v3.29.0 [CreateWithTemplate](https://github.com/pressly/goose/blob/v3.29.0/create.go) provides Version/CamelName template values and exclusive creation, but no returned path and possible partial template-error output, motivating staging. [Fix](https://github.com/pressly/goose/blob/v3.29.0/fix.go) uses os.Rename without aggregate maintenance or no-overwrite guarantees; reuse creation, not that reconciliation. [NewGoMigration](https://github.com/pressly/goose/blob/v3.29.0/migration.go) and [provider options](https://github.com/pressly/goose/blob/v3.29.0/provider_options.go) support fresh provider-local registration. [Provider operations](https://github.com/pressly/goose/blob/v3.29.0/provider.go) and [initialization](https://github.com/pressly/goose/blob/v3.29.0/provider_run.go) establish ownership and version-table behavior. Native [argument validation](https://github.com/urfave/cli/blob/v3.14.0/command_run.go) precedes Action. These sources were read locally; the implementation tests exercise integration and artifact exclusion on the local build. + +Reconciled against latest `origin/main` at `0034a561ba5eade4ee6fb29436c76e4e34fa3a56` plus this branch. Required local checks passed; PostgreSQL integrations skipped without DSNs. Keep implementation in progress until independent review and publication, then record the real PR URL. diff --git a/migrate/README.md b/migrate/README.md index f76e5e0..f2aa56e 100644 --- a/migrate/README.md +++ b/migrate/README.md @@ -5,21 +5,64 @@ It returns `*goose.Provider` directly: Goose owns execution, version tracking, status, and errors. There is no lifecycle dependency or background work. -Create SQL migration files with Goose's -[`create` command](https://pressly.github.io/goose/documentation/cli-commands/#create). -For timestamp-to-sequential reconciliation before deployment, use its -[`fix` command](https://pressly.github.io/goose/documentation/cli-commands/#fix) -as described in [hybrid versioning](https://github.com/pressly/goose#hybrid-versioning). -These commands edit source files before embedding; `fix` does not update database -history. Never renumber migrations already applied to shared databases. +Use the standalone `corestack-goose` development tool for SQL and Go creation +and timestamp-to-sequential reconciliation. Its generated Go migrations work +unchanged with this package's provider-local registration: -**Goose's default Go template is not compatible unchanged with this package.** -It registers migrations globally from `init`, while this package uses explicit -provider-local registration. Until the compatible authoring CLI planned in -[#9](https://gitea.auvem.com/corestack/go/issues/9) is available, Go migrations -require the adaptation described below; do not treat stock `goose create ... go` -output as ready to use. That follow-up must supply and document the supported -creation/reconciliation workflow before it is complete. +```sh +go install go.auvem.com/corestack/cmd/corestack-goose@ +mkdir -p migrations +corestack-goose create --dir ./migrations create_events sql +corestack-goose fix --dir ./migrations +corestack-goose create --dir ./migrations populate_events go +corestack-goose fix --dir ./migrations +# Or use the version selected by your consumer module: +go run go.auvem.com/corestack/cmd/corestack-goose create --dir ./migrations add_index sql +``` + +The directory must exist and contain regular migration files, with no symlink +components in its path (on macOS use `/private/tmp`, rather than `/tmp`). Names +are lowercase ASCII words separated by underscores. Creation uses Goose's stock +SQL template and a custom transactional Go template. Go files use the fixed +`__migration.go` suffix, so names such as `test`, `create_test` and +`windows` stay buildable on every platform. Edit the SQL or Go up/down function bodies; +leave factories and the generated `register.go` aggregate intact. +`GoMigrations()` supplies fresh registrations on each call. The aggregate exists +even for SQL-only sets. Versions are unique across SQL and Go; Goose timestamps +have second resolution, so fix a new migration before creating another in the +same second, or wait for a distinct timestamp. + +`fix` preserves sequential IDs, allocates timestamps after the highest sequential +ID with at least five-digit padding, and updates filenames and registration +literals together. It preserves migration bodies. It rejects unsupported Go +layouts, edited aggregates, duplicate versions/factories, unsafe paths, collisions, +overflow and lexical/numeric order disagreement before writing. It does not adapt +stock Goose Go output, whose global `init` registry is incompatible here. +For concepts, see Goose's [hybrid versioning](https://github.com/pressly/goose#hybrid-versioning) +and [SQL annotations](https://pressly.github.io/goose/documentation/annotations/). + +Before `fix`, commit or separately back up the **complete source set**, including +untracked migrations and tracked edits. Run one authoring process with no concurrent +editor changes. New destinations are exclusive; the tool stages output and fully +publishes each destination before removing its old source. This is **not an atomic +multi-file operation**. On a write/removal failure, stop and inspect the reported +completed, partial/current and outstanding paths. Restore the complete source set +and aggregate from that commit/backup before retrying; ordinary Git restore cannot +recover untracked files or unsaved edits. Interruption can leave duplicates, +aggregate drift or staging files; there is no automatic rollback or journal. + +Source reconciliation never rewrites database history. Use it before those IDs +reach a shared database; preserve deployed IDs and add migrations afterward. The +tool cannot determine deployment history. + +The standalone tool has no database setup. Production imports `migrate` and, +optionally, `migrate/lifecycle`; neither imports authoring code. No build tags are +needed. The generated consumer imports only Goose and the standard library. +The artifact test builds both executables, decodes production dependency identities, +and checks linker symbols for project authoring and Goose Create/CreateWithTemplate/ +Fix/SetSequential entrypoints. Goose's package remains a runtime dependency; +these checks establish the tested build's boundary, not a universal ban on file I/O +or proof about arbitrary downstream binaries. Build your application normally. Embed migrations in the application's binary and run them before serving traffic: @@ -56,21 +99,62 @@ not defer it when sharing the pool with the application. Use `"."` for SQL files at the embedded root. Files in the selected directory are discovered directly, without recursively searching subdirectories. -Preserve Goose's creator/fix filenames and ship the complete migration history. +Preserve the tool's creator/fix filenames and ship the complete migration history. The pinned Goose SQL-only provider relies on lexical filename order matching numeric version order. Timestamp versions and gaps are supported; never derive versions from file counts. For operations, errors, and SQL annotations, use the [Provider API](https://pkg.go.dev/github.com/pressly/goose/v3#Provider) and [SQL migration docs](https://pressly.github.io/goose/documentation/annotations/). -For Go functions, use Goose's -[provider-local registration](https://pressly.github.io/goose/documentation/provider/#withgomigrations): -pass fresh `goose.NewGoMigration` values as the trailing arguments to `migrate.New`. -The creator's default Go template instead calls `goose.AddMigrationContext` from -`init`; replace that registration with provider-local construction. Global -registration is disabled to keep each provider's migration set isolated and avoid -import side effects. If `fix` renames Go migrations, update their explicit -registration versions to match. Versions must be unique across SQL and Go files. +For generated Go functions, import the consumer's migration package and pass +`migrations.GoMigrations()...` as the trailing arguments to `migrate.New`. +Embed the complete directory with `//go:embed migrations` (and use a different +variable name, such as `migrationFiles`, from the imported package). The directory +must exist even for Go-only sets. Global registration is disabled; never share +mutable registration objects across providers. See Goose's +[provider-local registration](https://pressly.github.io/goose/documentation/provider/#withgomigrations). + +## Lifecycle startup and native commands + +`go.auvem.com/corestack/migrate/lifecycle` is a thin adapter; `migrate` itself has +no lifecycle dependency. Alias it `migratelifecycle` to distinguish the core +lifecycle package. Its resolver runs after the caller's database setup and should +construct a fresh provider with fresh registrations: + +```go +resolve := func(ctx context.Context) (*goose.Provider, error) { + return migrate.New(db, goose.DialectPostgres, migrationFiles, + "migrations", logger, migrations.GoMigrations()...) +} +startup, err := migratelifecycle.Module("migrations", migratelifecycle.Require, resolve) +if err != nil { + return err +} +root := &urfavecli.Command{Name: "app", Commands: []*urfavecli.Command{ + {Name: "serve", Action: cli.Serve(cfg, []lifecycle.Module{database, startup, server})}, + migratelifecycle.Commands(cfg, []lifecycle.Module{database}, resolve), +}} +code, err := cli.Run(ctx, root, args) // args includes argv[0]; caller owns process exit. +``` + +Here `database.Setup` opens `db` and cleans up its partial acquisition on failure; +`database.Teardown` closes it. Construct the tree and captured resource bindings +fresh for independent invocations. Declaration does no work. Select `Apply` to run +Up during setup, or `Require` to fail before later setup/services when migrations +are pending. Require is a point-in-time check without a deployment lock. It and +status may initialize Goose's version table; neither promises read-only DB access. +The migration module never closes the provider or shared database. + +The native group supplies `migrate up`, `up-to ` (inclusive), `down` +(one migration), `down-to ` and `status` (version/state/source table). +Pass exactly the database modules, excluding the startup migration module and +unrelated modules. Services are suppressed; the existing `lifecycle/cli.Action` +owns signals, cancellation, errors and reverse cleanup. Help and invalid arguments +never run setup. Target versions must be nonnegative decimal int64 values, with +no extra arguments; malformed usage returns exit 2 through `cli.Run`. Zero follows +Goose: `down-to 0` rolls everything back, while `up-to 0` is a runtime error. +Resolver/Goose/cleanup and status Writer failures return runtime errors (exit 1). +Supply the native command Writer and configured Goose logger for output. Postgres providers automatically use Goose's default session advisory locker. Concurrent startup calls from separate providers/processes serialize migration diff --git a/migrate/lifecycle/lifecycle.go b/migrate/lifecycle/lifecycle.go new file mode 100644 index 0000000..2fd2308 --- /dev/null +++ b/migrate/lifecycle/lifecycle.go @@ -0,0 +1,146 @@ +// Package lifecycle adapts Goose providers to startup policy and native commands. +// Callers own database modules and supply fresh, late-bound providers. +package lifecycle + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "text/tabwriter" + + "github.com/pressly/goose/v3" + urfavecli "github.com/urfave/cli/v3" + "go.auvem.com/corestack/lifecycle" + "go.auvem.com/corestack/lifecycle/cli" +) + +// Policy selects migration behavior during module setup. +type Policy string + +const ( + // Apply runs pending migrations before later modules start. + Apply Policy = "apply" + // Require fails setup when migrations are pending. This is a point-in-time + // check, not a deployment lock; Goose may initialize its version table. + Require Policy = "require" +) + +// Module declares startup migration work without resolving provider yet. +// Register it after the database module and before dependent modules. The +// resolver must return fresh registrations and clean up any partial acquisition. +// Teardown never closes the provider: the database module owns its database. +func Module(name string, policy Policy, provider func(context.Context) (*goose.Provider, error)) (lifecycle.Module, error) { + if name == "" || (policy != Apply && policy != Require) || provider == nil { + return lifecycle.Module{}, errors.New("migrate: name, apply/require policy and provider resolver are required") + } + return lifecycle.Module{Name: name, Teardown: func(context.Context) error { return nil }, Setup: func(ctx context.Context) error { + p, err := resolve(ctx, provider) + if err != nil { + return err + } + if policy == Apply { + _, err = p.Up(ctx) + } else { + var pending bool + pending, err = p.HasPending(ctx) + if err == nil && pending { + return errors.New("migrate: pending migrations; run migrate up before startup") + } + } + if err != nil { + return fmt.Errorf("migrate: %s: %w", policy, err) + } + return nil + }}, nil +} + +func resolve(ctx context.Context, provider func(context.Context) (*goose.Provider, error)) (*goose.Provider, error) { + if provider == nil { + return nil, errors.New("migrate: provider resolver is required") + } + p, err := provider(ctx) + if err != nil { + return nil, fmt.Errorf("migrate: resolve provider: %w", err) + } + if p == nil { + return nil, errors.New("migrate: resolver returned nil provider") + } + return p, nil +} + +// Commands returns a fresh native migrate group using exactly modules, with +// services suppressed. Supply database modules only, excluding startup policy. +// Resolvers run after setup. Help/usage starts nothing; Action owns signals and +// reverse cleanup. Use fresh trees and bindings for independent invocations. +// Status may initialize the version table; it does not run migration bodies. +func Commands(config lifecycle.Config, modules []lifecycle.Module, provider func(context.Context) (*goose.Provider, error)) *urfavecli.Command { + group := &urfavecli.Command{Name: "migrate", Usage: "Manage database migrations"} + for _, name := range []string{"up", "up-to", "down", "down-to", "status"} { + targeted := strings.HasSuffix(name, "-to") + leaf := &urfavecli.Command{Name: name, ArgValidator: func(_ context.Context, cmd *urfavecli.Command) error { + if targeted { + if cmd.NArg() == 1 { + if _, err := version(cmd.Args().First()); err == nil { + return nil + } + } + return urfavecli.Exit("expected one nonnegative decimal int64 version", 2) + } + if cmd.NArg() != 0 { + return urfavecli.Exit("unexpected positional arguments", 2) + } + return nil + }} + if targeted { + leaf.ArgsUsage = "" + } + leaf.Action = cli.Action(config, modules, func(ctx context.Context, cmd *urfavecli.Command) error { + p, err := resolve(ctx, provider) + if err != nil { + return err + } + target, _ := version(cmd.Args().First()) // ArgValidator checked targeted leaves before setup. + switch name { + case "up": + _, err = p.Up(ctx) + case "up-to": + _, err = p.UpTo(ctx, target) + case "down": + _, err = p.Down(ctx) + case "down-to": + _, err = p.DownTo(ctx, target) + case "status": + var statuses []*goose.MigrationStatus + statuses, err = p.Status(ctx) + if err == nil { + writer := tabwriter.NewWriter(cmd.Writer, 0, 4, 2, ' ', 0) + if _, err = fmt.Fprintln(writer, "VERSION\tSTATE\tSOURCE"); err == nil { + for _, status := range statuses { + if _, err = fmt.Fprintf(writer, "%d\t%s\t%s\n", status.Source.Version, status.State, status.Source.Path); err != nil { + break + } + } + } + if err == nil { + err = writer.Flush() + } + } + } + if err != nil { + return fmt.Errorf("migrate %s: %w", name, err) + } + return nil + }) + group.Commands = append(group.Commands, leaf) + } + return group +} + +func version(raw string) (int64, error) { + if raw == "" || strings.IndexFunc(raw, func(r rune) bool { return r < '0' || r > '9' }) >= 0 { + return 0, errors.New("invalid version") + } + return strconv.ParseInt(raw, 10, 64) +} diff --git a/migrate/lifecycle/lifecycle_test.go b/migrate/lifecycle/lifecycle_test.go new file mode 100644 index 0000000..111fa7c --- /dev/null +++ b/migrate/lifecycle/lifecycle_test.go @@ -0,0 +1,289 @@ +package lifecycle_test + +import ( + "bytes" + "context" + "database/sql" + "embed" + "errors" + "io" + "log/slog" + "os" + "path/filepath" + "reflect" + "strings" + "syscall" + "testing" + "time" + + "github.com/pressly/goose/v3" + urfavecli "github.com/urfave/cli/v3" + "go.auvem.com/corestack/lifecycle" + "go.auvem.com/corestack/lifecycle/cli" + "go.auvem.com/corestack/migrate" + adapter "go.auvem.com/corestack/migrate/lifecycle" + _ "modernc.org/sqlite" +) + +//go:embed testdata +var files embed.FS + +func config() lifecycle.Config { + return lifecycle.Config{Logger: slog.New(slog.NewTextHandler(io.Discard, nil))} +} + +func database(t *testing.T) *sql.DB { + t.Helper() + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := db.Close(); err != nil { + t.Error(err) + } + }) + return db +} +func provider(t *testing.T, db *sql.DB) *goose.Provider { + t.Helper() + p, err := migrate.New(db, goose.DialectSQLite3, files, "testdata/migrations", config().Logger) + if err != nil { + t.Fatal(err) + } + return p +} +func TestStartupPolicies(t *testing.T) { + for _, policy := range []adapter.Policy{adapter.Apply, adapter.Require} { + for _, applied := range []bool{false, true} { + t.Run(string(policy)+"/applied="+stringBool(applied), func(t *testing.T) { + db := database(t) + p := provider(t, db) + if applied { + if _, err := p.Up(t.Context()); err != nil { + t.Fatal(err) + } + } + var calls []string + var setupCtx context.Context + dbModule := lifecycle.Module{Name: "db", Setup: func(ctx context.Context) error { setupCtx = ctx; calls = append(calls, "db"); return nil }, Teardown: func(ctx context.Context) error { + if ctx.Err() != nil { + t.Error(ctx.Err()) + } + calls = append(calls, "close") + return nil + }} + module, err := adapter.Module("migrations", policy, func(ctx context.Context) (*goose.Provider, error) { + if ctx != setupCtx { + t.Error("setup context changed") + } + calls = append(calls, "resolve") + return provider(t, db), nil + }) + if err != nil { + t.Fatal(err) + } + if len(calls) != 0 || module.Run != nil { + t.Fatal("declaration started work") + } + app, err := lifecycle.New(config()) + if err != nil { + t.Fatal(err) + } + later := lifecycle.Module{Name: "later", Setup: func(context.Context) error { calls = append(calls, "later"); return nil }, Run: func(context.Context) error { calls = append(calls, "service"); return nil }, Teardown: func(context.Context) error { calls = append(calls, "later-close"); return nil }} + for _, m := range []lifecycle.Module{dbModule, module, later} { + if err := app.Register(m); err != nil { + t.Fatal(err) + } + } + err = app.Run(t.Context()) + if policy == adapter.Require && !applied { + if err == nil || !strings.Contains(err.Error(), "pending") { + t.Fatalf("pending Require: %v", err) + } + if !reflect.DeepEqual(calls, []string{"db", "resolve", "close"}) { + t.Fatal(calls) + } + } else { + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(calls, []string{"db", "resolve", "later", "service", "later-close", "close"}) { + t.Fatal(calls) + } + pending, err := p.HasPending(t.Context()) + if err != nil || pending { + t.Fatalf("pending=%v err=%v", pending, err) + } + } + if err := db.PingContext(t.Context()); err != nil { + t.Fatalf("adapter closed caller DB: %v", err) + } + }) + } + } +} +func stringBool(v bool) string { + if v { + return "true" + } + return "false" +} + +func TestModuleValidationAndErrors(t *testing.T) { + sentinel := errors.New("resolve failure") + for _, tc := range []struct { + name string + policy adapter.Policy + resolver func(context.Context) (*goose.Provider, error) + }{{"", adapter.Apply, func(context.Context) (*goose.Provider, error) { return nil, nil }}, {"m", "", func(context.Context) (*goose.Provider, error) { return nil, nil }}, {"m", "invalid", func(context.Context) (*goose.Provider, error) { return nil, nil }}, {"m", adapter.Apply, nil}} { + if _, err := adapter.Module(tc.name, tc.policy, tc.resolver); err == nil { + t.Fatal("accepted invalid module") + } + } + for _, policy := range []adapter.Policy{adapter.Apply, adapter.Require} { + for _, resolver := range []func(context.Context) (*goose.Provider, error){func(context.Context) (*goose.Provider, error) { return nil, nil }, func(context.Context) (*goose.Provider, error) { return nil, sentinel }, func(context.Context) (*goose.Provider, error) { + db := database(t) + p := provider(t, db) + if err := db.Close(); err != nil { + t.Fatal(err) + } + return p, nil + }} { + m, err := adapter.Module("m", policy, resolver) + if err != nil { + t.Fatal(err) + } + if err := m.Setup(t.Context()); err == nil { + t.Fatal("accepted resolver/DB failure") + } + } + } + m, err := adapter.Module("m", adapter.Apply, func(context.Context) (*goose.Provider, error) { return nil, sentinel }) + if err != nil { + t.Fatal(err) + } + if err := m.Setup(t.Context()); !errors.Is(err, sentinel) { + t.Fatal(err) + } +} + +func run(t *testing.T, args []string, modules []lifecycle.Module, resolver func(context.Context) (*goose.Provider, error), writer io.Writer) (int, error) { + t.Helper() + root := &urfavecli.Command{Name: "app", Writer: writer, ErrWriter: io.Discard, Commands: []*urfavecli.Command{adapter.Commands(config(), modules, resolver)}} + return cli.Run(t.Context(), root, append([]string{"app", "migrate"}, args...)) +} +func TestCommandsSQLite(t *testing.T) { + db := database(t) + var out bytes.Buffer + var calls []string + module := lifecycle.Module{Name: "db", Setup: func(context.Context) error { calls = append(calls, "setup"); return nil }, Run: func(context.Context) error { t.Error("one-off ran service"); return nil }, Teardown: func(context.Context) error { calls = append(calls, "cleanup"); return nil }} + resolver := func(context.Context) (*goose.Provider, error) { + calls = append(calls, "resolve") + return provider(t, db), nil + } + for _, step := range []struct { + args []string + version int64 + }{{[]string{"up-to", "1"}, 1}, {[]string{"up"}, 2}, {[]string{"status"}, 2}, {[]string{"down"}, 1}, {[]string{"up-to", "2"}, 2}, {[]string{"down-to", "0"}, 0}} { + calls = nil + code, err := run(t, step.args, []lifecycle.Module{module}, resolver, &out) + if code != 0 || err != nil { + t.Fatalf("%v code=%d err=%v", step.args, code, err) + } + version, err := provider(t, db).GetDBVersion(t.Context()) + if err != nil || version != step.version { + t.Fatalf("%v version=%d err=%v", step.args, version, err) + } + if !reflect.DeepEqual(calls, []string{"setup", "resolve", "cleanup"}) { + t.Fatal(calls) + } + } + // Zero is valid syntax; Goose UpTo rejects it after setup, unlike DownTo(0). + code, err := run(t, []string{"up-to", "0"}, []lifecycle.Module{module}, resolver, &out) + if code != 1 || err == nil || !strings.Contains(err.Error(), "greater than 0") { + t.Fatalf("zero UpTo: code=%d err=%v", code, err) + } + if !strings.Contains(out.String(), "VERSION") || !strings.Contains(out.String(), "applied") || !strings.Contains(out.String(), "00002_second.sql") { + t.Fatal(out.String()) + } +} +func TestHelpAndUsageWithoutSetup(t *testing.T) { + for _, leaf := range []string{"up", "up-to", "down", "down-to", "status"} { + code, err := run(t, []string{leaf, "--help"}, nil, nil, io.Discard) + if code != 0 || err != nil { + t.Fatalf("%s help: %d %v", leaf, code, err) + } + } + + module := lifecycle.Module{Name: "db", Setup: func(context.Context) error { t.Error("usage opened DB"); return nil }, Teardown: func(context.Context) error { return nil }} + for _, tc := range []struct { + args []string + code int + }{{nil, 0}, {[]string{"--help"}, 0}, {[]string{"help", "up-to"}, 0}, {[]string{"missing"}, 2}, {[]string{"up", "extra"}, 2}, {[]string{"down", "extra"}, 2}, {[]string{"status", "extra"}, 2}, {[]string{"up-to"}, 2}, {[]string{"down-to"}, 2}, {[]string{"up-to", "1", "2"}, 2}, {[]string{"down-to", "--", "-1"}, 2}, {[]string{"up-to", "+1"}, 2}, {[]string{"up-to", "9223372036854775808"}, 2}, {[]string{"down-to", "1x"}, 2}, {[]string{"up-to", "1"}, 2}} { + code, err := run(t, tc.args, []lifecycle.Module{module}, nil, io.Discard) + if code != tc.code || (code == 0 && err != nil) || (code != 0 && err == nil) { + t.Fatalf("args=%v code=%d err=%v", tc.args, code, err) + } + } +} + +type brokenWriter struct{ err error } + +func (w brokenWriter) Write([]byte) (int, error) { return 0, w.err } +func TestCommandFailureCleanup(t *testing.T) { + db := database(t) + writeErr := errors.New("write failed") + resolveErr := errors.New("resolve failed") + cleanupErr := errors.New("cleanup failed") + for _, tc := range []struct { + name string + resolver func(context.Context) (*goose.Provider, error) + writer io.Writer + want error + }{{"writer", func(context.Context) (*goose.Provider, error) { return provider(t, db), nil }, brokenWriter{writeErr}, writeErr}, {"resolver", func(context.Context) (*goose.Provider, error) { return nil, resolveErr }, io.Discard, resolveErr}, {"nil", nil, io.Discard, nil}, {"nil-provider", func(context.Context) (*goose.Provider, error) { return nil, nil }, io.Discard, nil}} { + t.Run(tc.name, func(t *testing.T) { + cleaned := false + module := lifecycle.Module{Name: "db", Setup: func(context.Context) error { return nil }, Teardown: func(context.Context) error { cleaned = true; return cleanupErr }} + code, err := run(t, []string{"status"}, []lifecycle.Module{module}, tc.resolver, tc.writer) + if code != 1 || err == nil || !cleaned || !errors.Is(err, cleanupErr) || (tc.want != nil && !errors.Is(err, tc.want)) { + t.Fatalf("code=%d err=%v cleaned=%v", code, err, cleaned) + } + }) + } +} +func TestCommandSignalContext(t *testing.T) { + signals := make(chan os.Signal, 2) + cfg := config() + cfg.Signals = signals + entered := make(chan struct{}) + done := make(chan error, 1) + cleaned := false + module := lifecycle.Module{Name: "db", Setup: func(context.Context) error { return nil }, Teardown: func(ctx context.Context) error { + if ctx.Err() != nil { + t.Error(ctx.Err()) + } + cleaned = true + return nil + }} + root := &urfavecli.Command{Name: "app", Writer: io.Discard, ErrWriter: io.Discard, Commands: []*urfavecli.Command{adapter.Commands(cfg, []lifecycle.Module{module}, func(ctx context.Context) (*goose.Provider, error) { + close(entered) + <-ctx.Done() + return nil, ctx.Err() + })}} + go func() { _, err := cli.Run(t.Context(), root, []string{"app", "migrate", "up"}); done <- err }() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("resolver did not start") + } + signals <- syscall.SIGTERM + select { + case err := <-done: + if err != nil || !cleaned { + t.Fatalf("err=%v cleaned=%v", err, cleaned) + } + case <-time.After(5 * time.Second): + t.Fatal("signal did not cancel resolver") + } +} diff --git a/migrate/lifecycle/testdata/migrations/00001_first.sql b/migrate/lifecycle/testdata/migrations/00001_first.sql new file mode 100644 index 0000000..a05cde0 --- /dev/null +++ b/migrate/lifecycle/testdata/migrations/00001_first.sql @@ -0,0 +1,4 @@ +-- +goose Up +CREATE TABLE events (id INTEGER PRIMARY KEY); +-- +goose Down +DROP TABLE events; diff --git a/migrate/lifecycle/testdata/migrations/00002_second.sql b/migrate/lifecycle/testdata/migrations/00002_second.sql new file mode 100644 index 0000000..143fcdb --- /dev/null +++ b/migrate/lifecycle/testdata/migrations/00002_second.sql @@ -0,0 +1,4 @@ +-- +goose Up +INSERT INTO events VALUES (2); +-- +goose Down +DELETE FROM events WHERE id = 2; diff --git a/migrate/migrate.go b/migrate/migrate.go index a0778c2..1270417 100644 --- a/migrate/migrate.go +++ b/migrate/migrate.go @@ -23,10 +23,13 @@ import ( // across providers/processes. PostgreSQL connections must preserve session // affinity (transaction-pooling proxies cannot safely hold session locks). // -// Use the returned Provider's Up, UpTo, DownTo, Status, GetDBVersion, GetVersions, -// and HasPending methods directly. Create migration files with Goose's create -// command; lexical filename order must match numeric version order. Go migrations -// are registered per provider; global Go registration is ignored. +// Use the returned Provider's Up, UpTo, Down, DownTo, Status, GetDBVersion, GetVersions, +// and HasPending methods directly. Create and reconcile sources with the standalone +// corestack-goose tool (go run go.auvem.com/corestack/cmd/corestack-goose). +// Pass migrations.GoMigrations()... from its generated package; stock Goose Go +// output uses an incompatible global init registry. Lexical filename order must +// match numeric version order. Each provider needs fresh Go registrations; +// global Go registration is ignored. func New(db *sql.DB, dialect goose.Dialect, files embed.FS, dir string, logger *slog.Logger, migrations ...*goose.Migration) (*goose.Provider, error) { for i, migration := range migrations { if migration == nil {