// SPDX-License-Identifier: AGPL-3.0-only package deploy import ( "context" "errors" "fmt" "os" "path/filepath" "strings" "testing" "time" "gamertan.com/tend/internal/config" "gamertan.com/tend/internal/eventlog" "gamertan.com/tend/internal/state" ) type fakeOperator struct { failReload bool failReloadAt int reloads int failRestartUnit string failRestartOnce bool rejectMarkers bool active map[string]bool starts, stops, restarts []string probes []string publicProbes []string failPublic bool failPublicAfter int candidateEnvironment map[string]string candidateFile string } func (f *fakeOperator) Restart(_ context.Context, unit string) error { f.restarts = append(f.restarts, unit) if unit == f.failRestartUnit { if f.failRestartOnce { f.failRestartUnit = "" } return errors.New("injected restart failure") } f.active[unit] = true return nil } func (f *fakeOperator) Stop(_ context.Context, unit string) error { f.stops = append(f.stops, unit) f.active[unit] = false return nil } func (f *fakeOperator) IsActive(_ context.Context, unit string) (bool, error) { return f.active[unit], nil } func (f *fakeOperator) StartCandidate(_ context.Context, unit, binary, environmentFile string, env map[string]string) error { if !filepath.IsAbs(binary) || !filepath.IsAbs(environmentFile) || len(env) == 0 { return errors.New("bad candidate") } f.starts = append(f.starts, unit) f.candidateFile = environmentFile f.candidateEnvironment = make(map[string]string, len(env)) for key, value := range env { f.candidateEnvironment[key] = value } f.active[unit] = true return nil } func (f *fakeOperator) ProbeURL(_ context.Context, value, contains string) error { f.publicProbes = append(f.publicProbes, value) if f.failPublic || (f.failPublicAfter > 0 && len(f.publicProbes) >= f.failPublicAfter) { return errors.New("injected public smoke failure") } if f.rejectMarkers && contains != "" { return errors.New("unexpected future-release smoke marker") } return nil } func TestPublicSmokeFailureRestoresBlueGreenHandlerAndSlot(t *testing.T) { cfg, old, fresh := baseConfig(t, "blue_green") handler := filepath.Join(cfg.Deployment.Root, "handler.caddy") template := filepath.Join(cfg.Deployment.Root, "handler.template") original := []byte("reverse_proxy 127.0.0.1:8090\n") _ = os.WriteFile(handler, original, 0o644) _ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644) blue := filepath.Join(cfg.Deployment.Root, "slots", "blue") green := filepath.Join(cfg.Deployment.Root, "slots", "green") _ = replaceSymlink(blue, old) _ = replaceSymlink(green, old) cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}} operator := &fakeOperator{active: map[string]bool{}, failPublic: true} if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true}); err == nil { t.Fatal("expected public smoke failure") } body, _ := os.ReadFile(handler) if string(body) != string(original) { t.Fatalf("handler not restored: %q", body) } target, err := resolveReleaseLink(cfg.Deployment.Root, green) if err != nil || target != old { t.Fatalf("green=%q err=%v", target, err) } record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) if err != nil { t.Fatal(err) } if record.ActiveRelease != old || record.LastAttemptOutcome != "failed" || record.CandidateRelease != "" || record.LastAttemptRelease != fresh { t.Fatalf("failed attempt state=%+v", record) } } func TestContinuityFailureRestoresBlueGreenRoute(t *testing.T) { cfg, old, fresh := baseConfig(t, "blue_green") handler := filepath.Join(cfg.Deployment.Root, "handler.caddy") template := filepath.Join(cfg.Deployment.Root, "handler.template") original := []byte("reverse_proxy 127.0.0.1:8090\n") _ = os.WriteFile(handler, original, 0o644) _ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644) blue := filepath.Join(cfg.Deployment.Root, "slots", "blue") green := filepath.Join(cfg.Deployment.Root, "slots", "green") _ = replaceSymlink(blue, old) _ = replaceSymlink(green, old) cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}} operator := &fakeOperator{active: map[string]bool{}, failPublicAfter: 3} if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err == nil || !strings.Contains(err.Error(), "continuity") { t.Fatalf("expected continuity failure, got %v", err) } body, _ := os.ReadFile(handler) if string(body) != string(original) { t.Fatalf("handler not restored: %q", body) } target, err := resolveReleaseLink(cfg.Deployment.Root, green) if err != nil || target != old { t.Fatalf("green=%q err=%v", target, err) } } func (f *fakeOperator) ValidateCaddy(context.Context, string) error { return nil } func (f *fakeOperator) ReloadCaddy(context.Context) error { f.reloads++ if f.failReload || (f.failReloadAt > 0 && f.reloads == f.failReloadAt) { return errors.New("injected reload failure") } return nil } func (f *fakeOperator) Probe(_ context.Context, address, host, path, contains string) error { f.probes = append(f.probes, address+path) if f.rejectMarkers && contains != "" { return errors.New("unexpected future-release smoke marker") } return nil } func baseConfig(t *testing.T, strategy string) (config.Config, string, string) { t.Helper() root := filepath.Join(t.TempDir(), "service") if err := os.MkdirAll(filepath.Join(root, "releases"), 0o755); err != nil { t.Fatal(err) } old := filepath.Join(root, "releases", "sha256-"+strings.Repeat("c", 64)) fresh := filepath.Join(root, "releases", "sha256-"+strings.Repeat("a", 64)) for _, dir := range []string{old, fresh} { if err := os.Mkdir(dir, 0o755); err != nil { t.Fatal(err) } if err := os.WriteFile(filepath.Join(dir, "app"), []byte("x"), 0o755); err != nil { t.Fatal(err) } } cfg := config.Config{SchemaVersion: 2, Service: config.Service{Name: "example-site", AllowedHost: "example.test", EnvironmentFile: "/etc/tend/environment/example-site.env"}, Build: config.Build{Package: "./cmd/site", Binary: "app", Branch: "main"}, Deployment: config.Deployment{Strategy: strategy, Root: root, LockFile: filepath.Join(root, "deploy.lock"), StateFile: filepath.Join(root, "state.json"), EventLog: filepath.Join(root, "deployment-events.jsonl"), HealthPath: "/healthz", ReadinessPath: "/readyz", CandidateTimeoutSecs: 2, ActivationWindowSecs: 1, Smoke: []config.Smoke{{Path: "/", Contains: "Example"}}, PublicSmoke: []config.PublicSmoke{{URL: "https://example.test/", Contains: "Example"}}}} return cfg, old, fresh } func manager(operator Operator, fresh string) Manager { return Manager{Operator: operator, Now: func() time.Time { return time.Unix(100, 0).UTC() }, Prepare: func(config.Config, string, string, string) (string, error) { return fresh, nil }, Inspect: func(config.Config, string, string, string) error { return nil }, ReadIdentity: func(string) (releaseIdentity, error) { return releaseIdentity{Version: "v0.2.0-preview.1", Commit: strings.Repeat("b", 40)}, nil }, OperationID: func() (string, error) { return strings.Repeat("d", 32), nil }, Sleep: func(context.Context, time.Duration) error { return nil }} } func singletonSettings(t *testing.T, cfg config.Config, currentRelease string) (*config.Singleton, string) { t.Helper() handler := filepath.Join(cfg.Deployment.Root, "handler.caddy") template := filepath.Join(cfg.Deployment.Root, "handler.template") if err := os.WriteFile(handler, []byte("reverse_proxy 127.0.0.1:8092\n"), 0o640); err != nil { t.Fatal(err) } if err := os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644); err != nil { t.Fatal(err) } current := filepath.Join(cfg.Deployment.Root, "current") if err := replaceSymlink(current, currentRelease); err != nil { t.Fatal(err) } return &config.Singleton{ Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN", CurrentLink: current, PreviousLink: filepath.Join(cfg.Deployment.Root, "previous"), CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, }, handler } func TestBlueGreenActivationAndRollback(t *testing.T) { cfg, old, fresh := baseConfig(t, "blue_green") handler := filepath.Join(cfg.Deployment.Root, "handler.caddy") template := filepath.Join(cfg.Deployment.Root, "handler.template") if err := os.WriteFile(handler, []byte("reverse_proxy 127.0.0.1:8090\n"), 0o640); err != nil { t.Fatal(err) } if err := os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644); err != nil { t.Fatal(err) } blue := filepath.Join(cfg.Deployment.Root, "slots", "blue") green := filepath.Join(cfg.Deployment.Root, "slots", "green") if err := replaceSymlink(blue, old); err != nil { t.Fatal(err) } if err := replaceSymlink(green, old); err != nil { t.Fatal(err) } cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}} if err := cfg.Validate(); err != nil { t.Fatal(err) } operator := &fakeOperator{active: map[string]bool{"example-blue.service": true, "example-green.service": true}} m := manager(operator, fresh) var events []eventlog.Event m.AppendEvent = func(_ string, event eventlog.Event) error { events = append(events, event) return nil } report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}) if err != nil { t.Fatal(err) } if report.ActiveRelease != fresh || report.PreviousRelease != old { t.Fatalf("report=%+v", report) } deployed, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) if err != nil { t.Fatal(err) } if deployed.DesiredRelease != fresh || deployed.CandidateRelease != "" || deployed.LastAttemptRelease != fresh || deployed.LastAttemptOutcome != "succeeded" { t.Fatalf("deployment identity state=%+v", deployed) } if len(events) != 2 || events[0].Phase != "candidate" || events[0].Outcome != "running" || events[1].Phase != "activation" || events[1].Outcome != "succeeded" || events[0].OperationID != events[1].OperationID { t.Fatalf("events=%+v", events) } if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 { t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err) } if target, err := resolveReleaseLink(cfg.Deployment.Root, green); err != nil || target != fresh { t.Fatalf("green=%q err=%v", target, err) } operator.rejectMarkers = true operator.probes = nil record, err := m.Rollback(context.Background(), cfg) if err != nil { t.Fatal(err) } if record.ActiveRelease != old || record.PreviousRelease != fresh { t.Fatalf("rollback=%+v", record) } if len(events) != 4 || events[2].Phase != "rollback" || events[2].Outcome != "running" || events[3].Phase != "rollback" || events[3].Outcome != "succeeded" || events[2].OperationID != events[3].OperationID || events[2].ArtifactDigest != strings.Repeat("c", 64) { t.Fatalf("rollback events=%+v", events) } if len(operator.probes) != 4 { t.Fatalf("rollback probes=%#v", operator.probes) } for _, probe := range operator.probes { if !strings.HasSuffix(probe, "/healthz") && !strings.HasSuffix(probe, "/readyz") { t.Fatalf("rollback applied future-release smoke checks: %#v", operator.probes) } } } func TestDeploymentEvidenceCanNeverBlockActivationOrRollback(t *testing.T) { cfg, old, fresh := baseConfig(t, "singleton_candidate") cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old) m := manager(&fakeOperator{active: map[string]bool{}}, fresh) appendCalls := 0 m.AppendEvent = func(string, eventlog.Event) error { appendCalls++ return errors.New("injected event failure") } report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}) if err != nil { t.Fatal(err) } if report.EventWarnings != 2 || report.ActiveRelease != fresh { t.Fatalf("report=%+v", report) } if _, err := m.Rollback(context.Background(), cfg); err != nil { t.Fatal(err) } record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) if err != nil || record.ActiveRelease != old || appendCalls != 4 { t.Fatalf("record=%+v appends=%d err=%v", record, appendCalls, err) } } func TestDeploymentEvidenceIdentityAndEntropyAreBestEffort(t *testing.T) { for _, test := range []struct { name string damage func(*Manager) }{ {"identity", func(manager *Manager) { manager.ReadIdentity = func(string) (releaseIdentity, error) { return releaseIdentity{}, errors.New("injected identity failure") } }}, {"entropy", func(manager *Manager) { manager.OperationID = func() (string, error) { return "", errors.New("injected entropy failure") } }}, } { t.Run(test.name, func(t *testing.T) { cfg, old, fresh := baseConfig(t, "singleton_candidate") cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old) manager := manager(&fakeOperator{active: map[string]bool{}}, fresh) test.damage(&manager) report, err := manager.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}) if err != nil || report.EventWarnings != 1 || report.ActiveRelease != fresh { t.Fatalf("report=%+v err=%v", report, err) } }) } } func TestBlueGreenCaddyFailureRestoresHandlerAndSlot(t *testing.T) { cfg, old, fresh := baseConfig(t, "blue_green") handler := filepath.Join(cfg.Deployment.Root, "handler.caddy") template := filepath.Join(cfg.Deployment.Root, "handler.template") original := []byte("reverse_proxy 127.0.0.1:8090\n") _ = os.WriteFile(handler, original, 0o644) _ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644) blue := filepath.Join(cfg.Deployment.Root, "slots", "blue") green := filepath.Join(cfg.Deployment.Root, "slots", "green") _ = replaceSymlink(blue, old) _ = replaceSymlink(green, old) cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}} operator := &fakeOperator{active: map[string]bool{}, failReload: true} m := manager(operator, fresh) if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil { t.Fatal("expected failure") } body, _ := os.ReadFile(handler) if string(body) != string(original) { t.Fatalf("handler not restored: %q", body) } target, err := resolveReleaseLink(cfg.Deployment.Root, green) if err != nil || target != old { t.Fatalf("green=%q err=%v", target, err) } failed, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) if err != nil { t.Fatal(err) } if failed.ActiveRelease != old || failed.LastAttemptOutcome != "failed" || failed.CandidateRelease != "" { t.Fatalf("failed state=%+v", failed) } } func TestSingletonRestartFailureRestoresPointers(t *testing.T) { cfg, old, fresh := baseConfig(t, "singleton_candidate") settings, handler := singletonSettings(t, cfg, old) cfg.Deployment.Singleton = settings operator := &fakeOperator{active: map[string]bool{}, failRestartUnit: "example-site.service", failRestartOnce: true} m := manager(operator, fresh) if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil { t.Fatal("expected failure") } target, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink) if err != nil || target != old { t.Fatalf("current=%q err=%v", target, err) } if _, err := os.Lstat(cfg.Deployment.Singleton.PreviousLink); !os.IsNotExist(err) { t.Fatal("previous pointer was not restored") } body, err := os.ReadFile(handler) if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" { t.Fatalf("handler=%q err=%v", body, err) } if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 { t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err) } if operator.active["example-site-tend-candidate.service"] { t.Fatal("candidate was not stopped after successful restoration") } } func TestStateRecordsSuccessfulActivation(t *testing.T) { cfg, old, fresh := baseConfig(t, "singleton_candidate") settings, handler := singletonSettings(t, cfg, old) cfg.Deployment.Singleton = settings operator := &fakeOperator{active: map[string]bool{}} m := manager(operator, fresh) if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err != nil { t.Fatal(err) } record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) if err != nil { t.Fatal(err) } if record.ActiveRelease != fresh || record.PreviousRelease != old { t.Fatalf("state=%+v", record) } if operator.candidateFile != cfg.Service.EnvironmentFile || len(operator.candidateEnvironment) != 1 || operator.candidateEnvironment["EXAMPLE_LISTEN"] != "127.0.0.1:18092" { t.Fatalf("candidate file=%q environment=%#v", operator.candidateFile, operator.candidateEnvironment) } body, err := os.ReadFile(handler) if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" { t.Fatalf("handler=%q err=%v", body, err) } if operator.reloads != 2 || operator.active["example-site-tend-candidate.service"] { t.Fatalf("reloads=%d active=%#v", operator.reloads, operator.active) } } func TestSingletonContinuityFailureRestoresHandlerPointersAndService(t *testing.T) { cfg, old, fresh := baseConfig(t, "singleton_candidate") settings, handler := singletonSettings(t, cfg, old) cfg.Deployment.Singleton = settings operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failPublicAfter: 4} m := manager(operator, fresh) if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil || !strings.Contains(err.Error(), "continuity") { t.Fatalf("expected continuity failure, got %v", err) } current, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink) if err != nil || current != old { t.Fatalf("current=%q err=%v", current, err) } body, err := os.ReadFile(handler) if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" { t.Fatalf("handler=%q err=%v", body, err) } if operator.active["example-site-tend-candidate.service"] { t.Fatal("candidate was not stopped after continuity restoration") } } func TestSingletonCaddyReloadFailuresRestorePriorRoute(t *testing.T) { for _, reload := range []int{1, 2} { t.Run(fmt.Sprintf("reload-%d", reload), func(t *testing.T) { cfg, old, fresh := baseConfig(t, "singleton_candidate") settings, handler := singletonSettings(t, cfg, old) cfg.Deployment.Singleton = settings operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failReloadAt: reload} if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true}); err == nil || !strings.Contains(err.Error(), "Caddy reload failed") { t.Fatalf("expected Caddy reload failure, got %v", err) } current, err := resolveReleaseLink(cfg.Deployment.Root, settings.CurrentLink) if err != nil || current != old { t.Fatalf("current=%q err=%v", current, err) } body, err := os.ReadFile(handler) if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" { t.Fatalf("handler=%q err=%v", body, err) } if operator.active["example-site-tend-candidate.service"] { t.Fatal("candidate was not stopped after route restoration") } }) } } func TestSingletonIncompleteRecoveryKeepsProvenCandidateRunning(t *testing.T) { cfg, old, fresh := baseConfig(t, "singleton_candidate") settings, _ := singletonSettings(t, cfg, old) cfg.Deployment.Singleton = settings operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failReload: true} _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true}) if err == nil || !strings.Contains(err.Error(), "candidate remains routed for operator recovery") { t.Fatalf("expected explicit incomplete recovery, got %v", err) } if !operator.active["example-site-tend-candidate.service"] { t.Fatal("proven candidate was stopped despite incomplete route restoration") } }