tend/internal/deploy/deploy_test.go

// SPDX-License-Identifier: AGPL-3.0-only

package deploy

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"gamertan.com/tend/internal/config"
	"gamertan.com/tend/internal/eventlog"
	"gamertan.com/tend/internal/state"
)

const testCandidateUnit = "example-site-tend-candidate-dddddddddddd.service"

func storeTestCandidateLease(t *testing.T, cfg config.Config, release, startedAt string) {
	t.Helper()
	lease := state.CandidateLease{SchemaVersion: state.CandidateLeaseSchemaVersion, Service: cfg.Service.Name, OperationID: strings.Repeat("d", 32), Release: release, Unit: testCandidateUnit, Address: cfg.Deployment.Singleton.CandidateAddress, StartedAt: startedAt}
	if err := state.StoreCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, lease); err != nil {
		t.Fatal(err)
	}
}

type fakeOperator struct {
	failReload              bool
	failReloadAt            int
	reloads                 int
	failRestartUnit         string
	failRestartOnce         bool
	rejectMarkers           bool
	active                  map[string]bool
	starts, stops, restarts []string
	probes                  []string
	publicProbes            []string
	failPublic              bool
	failPublicAfter         int
	candidateEnvironment    map[string]string
	candidateFile           string
}

func (f *fakeOperator) Restart(_ context.Context, unit string) error {
	f.restarts = append(f.restarts, unit)
	if unit == f.failRestartUnit {
		if f.failRestartOnce {
			f.failRestartUnit = ""
		}
		return errors.New("injected restart failure")
	}
	f.active[unit] = true
	return nil
}
func (f *fakeOperator) Stop(_ context.Context, unit string) error {
	f.stops = append(f.stops, unit)
	f.active[unit] = false
	return nil
}
func (f *fakeOperator) IsActive(_ context.Context, unit string) (bool, error) {
	return f.active[unit], nil
}
func (f *fakeOperator) StartCandidate(_ context.Context, unit, binary, environmentFile string, env map[string]string) error {
	if !filepath.IsAbs(binary) || !filepath.IsAbs(environmentFile) || len(env) == 0 {
		return errors.New("bad candidate")
	}
	f.starts = append(f.starts, unit)
	f.candidateFile = environmentFile
	f.candidateEnvironment = make(map[string]string, len(env))
	for key, value := range env {
		f.candidateEnvironment[key] = value
	}
	f.active[unit] = true
	return nil
}
func (f *fakeOperator) ProbeURL(_ context.Context, value, contains string) error {
	f.publicProbes = append(f.publicProbes, value)
	if f.failPublic || (f.failPublicAfter > 0 && len(f.publicProbes) >= f.failPublicAfter) {
		return errors.New("injected public smoke failure")
	}
	if f.rejectMarkers && contains != "" {
		return errors.New("unexpected future-release smoke marker")
	}
	return nil
}

func TestPublicSmokeFailureRestoresBlueGreenHandlerAndSlot(t *testing.T) {
	cfg, old, fresh := baseConfig(t, "blue_green")
	handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
	template := filepath.Join(cfg.Deployment.Root, "handler.template")
	original := []byte("reverse_proxy 127.0.0.1:8090\n")
	_ = os.WriteFile(handler, original, 0o644)
	_ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644)
	blue := filepath.Join(cfg.Deployment.Root, "slots", "blue")
	green := filepath.Join(cfg.Deployment.Root, "slots", "green")
	_ = replaceSymlink(blue, old)
	_ = replaceSymlink(green, old)
	cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}}
	operator := &fakeOperator{active: map[string]bool{}, failPublic: true}
	if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true}); err == nil {
		t.Fatal("expected public smoke failure")
	}
	body, _ := os.ReadFile(handler)
	if string(body) != string(original) {
		t.Fatalf("handler not restored: %q", body)
	}
	target, err := resolveReleaseLink(cfg.Deployment.Root, green)
	if err != nil || target != old {
		t.Fatalf("green=%q err=%v", target, err)
	}
	record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
	if err != nil {
		t.Fatal(err)
	}
	if record.ActiveRelease != old || record.LastAttemptOutcome != "failed" || record.CandidateRelease != "" || record.LastAttemptRelease != fresh {
		t.Fatalf("failed attempt state=%+v", record)
	}
}

func TestContinuityFailureRestoresBlueGreenRoute(t *testing.T) {
	cfg, old, fresh := baseConfig(t, "blue_green")
	handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
	template := filepath.Join(cfg.Deployment.Root, "handler.template")
	original := []byte("reverse_proxy 127.0.0.1:8090\n")
	_ = os.WriteFile(handler, original, 0o644)
	_ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644)
	blue := filepath.Join(cfg.Deployment.Root, "slots", "blue")
	green := filepath.Join(cfg.Deployment.Root, "slots", "green")
	_ = replaceSymlink(blue, old)
	_ = replaceSymlink(green, old)
	cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}}
	operator := &fakeOperator{active: map[string]bool{}, failPublicAfter: 3}
	if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err == nil || !strings.Contains(err.Error(), "continuity") {
		t.Fatalf("expected continuity failure, got %v", err)
	}
	body, _ := os.ReadFile(handler)
	if string(body) != string(original) {
		t.Fatalf("handler not restored: %q", body)
	}
	target, err := resolveReleaseLink(cfg.Deployment.Root, green)
	if err != nil || target != old {
		t.Fatalf("green=%q err=%v", target, err)
	}
}
func (f *fakeOperator) ValidateCaddy(context.Context, string) error { return nil }
func (f *fakeOperator) ReloadCaddy(context.Context) error {
	f.reloads++
	if f.failReload || (f.failReloadAt > 0 && f.reloads == f.failReloadAt) {
		return errors.New("injected reload failure")
	}
	return nil
}
func (f *fakeOperator) Probe(_ context.Context, address, host, path, contains string) error {
	f.probes = append(f.probes, address+path)
	if f.rejectMarkers && contains != "" {
		return errors.New("unexpected future-release smoke marker")
	}
	return nil
}

func baseConfig(t *testing.T, strategy string) (config.Config, string, string) {
	t.Helper()
	root := filepath.Join(t.TempDir(), "service")
	if err := os.MkdirAll(filepath.Join(root, "releases"), 0o755); err != nil {
		t.Fatal(err)
	}
	old := filepath.Join(root, "releases", "sha256-"+strings.Repeat("c", 64))
	fresh := filepath.Join(root, "releases", "sha256-"+strings.Repeat("a", 64))
	for _, dir := range []string{old, fresh} {
		if err := os.Mkdir(dir, 0o755); err != nil {
			t.Fatal(err)
		}
		if err := os.WriteFile(filepath.Join(dir, "app"), []byte("x"), 0o755); err != nil {
			t.Fatal(err)
		}
	}
	cfg := config.Config{SchemaVersion: 2, Service: config.Service{Name: "example-site", AllowedHost: "example.test", EnvironmentFile: "/etc/tend/environment/example-site.env"}, Build: config.Build{Package: "./cmd/site", Binary: "app", Branch: "main"}, Deployment: config.Deployment{Strategy: strategy, Root: root, LockFile: filepath.Join(root, "deploy.lock"), StateFile: filepath.Join(root, "state.json"), EventLog: filepath.Join(root, "deployment-events.jsonl"), HealthPath: "/healthz", ReadinessPath: "/readyz", CandidateTimeoutSecs: 2, ActivationWindowSecs: 1, Smoke: []config.Smoke{{Path: "/", Contains: "Example"}}, PublicSmoke: []config.PublicSmoke{{URL: "https://example.test/", Contains: "Example"}}}}
	return cfg, old, fresh
}
func manager(operator Operator, fresh string) Manager {
	return Manager{Operator: operator, Now: func() time.Time { return time.Unix(100, 0).UTC() }, Prepare: func(config.Config, string, string, string) (string, error) { return fresh, nil }, Inspect: func(config.Config, string, string, string) error { return nil }, ReadIdentity: func(string) (releaseIdentity, error) {
		return releaseIdentity{Version: "v0.2.0-preview.1", Commit: strings.Repeat("b", 40)}, nil
	}, OperationID: func() (string, error) { return strings.Repeat("d", 32), nil }, Sleep: func(context.Context, time.Duration) error { return nil }}
}

func singletonSettings(t *testing.T, cfg config.Config, currentRelease string) (*config.Singleton, string) {
	t.Helper()
	handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
	template := filepath.Join(cfg.Deployment.Root, "handler.template")
	if err := os.WriteFile(handler, []byte("reverse_proxy 127.0.0.1:8092\n"), 0o640); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	current := filepath.Join(cfg.Deployment.Root, "current")
	if err := replaceSymlink(current, currentRelease); err != nil {
		t.Fatal(err)
	}
	return &config.Singleton{
		Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN",
		CurrentLink: current, PreviousLink: filepath.Join(cfg.Deployment.Root, "previous"), CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"),
		CaddyHandler: handler, CaddyHandlerTemplate: template,
	}, handler
}

func TestBlueGreenActivationAndRollback(t *testing.T) {
	cfg, old, fresh := baseConfig(t, "blue_green")
	handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
	template := filepath.Join(cfg.Deployment.Root, "handler.template")
	if err := os.WriteFile(handler, []byte("reverse_proxy 127.0.0.1:8090\n"), 0o640); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644); err != nil {
		t.Fatal(err)
	}
	blue := filepath.Join(cfg.Deployment.Root, "slots", "blue")
	green := filepath.Join(cfg.Deployment.Root, "slots", "green")
	if err := replaceSymlink(blue, old); err != nil {
		t.Fatal(err)
	}
	if err := replaceSymlink(green, old); err != nil {
		t.Fatal(err)
	}
	cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}}
	if err := cfg.Validate(); err != nil {
		t.Fatal(err)
	}
	operator := &fakeOperator{active: map[string]bool{"example-blue.service": true, "example-green.service": true}}
	m := manager(operator, fresh)
	var events []eventlog.Event
	m.AppendEvent = func(_ string, event eventlog.Event) error {
		events = append(events, event)
		return nil
	}
	report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
	if err != nil {
		t.Fatal(err)
	}
	if report.ActiveRelease != fresh || report.PreviousRelease != old {
		t.Fatalf("report=%+v", report)
	}
	deployed, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
	if err != nil {
		t.Fatal(err)
	}
	if deployed.DesiredRelease != fresh || deployed.CandidateRelease != "" || deployed.LastAttemptRelease != fresh || deployed.LastAttemptOutcome != "succeeded" {
		t.Fatalf("deployment identity state=%+v", deployed)
	}
	if len(events) != 2 || events[0].Phase != "candidate" || events[0].Outcome != "running" || events[1].Phase != "activation" || events[1].Outcome != "succeeded" || events[0].OperationID != events[1].OperationID {
		t.Fatalf("events=%+v", events)
	}
	if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 {
		t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err)
	}
	if target, err := resolveReleaseLink(cfg.Deployment.Root, green); err != nil || target != fresh {
		t.Fatalf("green=%q err=%v", target, err)
	}
	operator.rejectMarkers = true
	operator.probes = nil
	record, err := m.Rollback(context.Background(), cfg)
	if err != nil {
		t.Fatal(err)
	}
	if record.ActiveRelease != old || record.PreviousRelease != fresh {
		t.Fatalf("rollback=%+v", record)
	}
	if len(events) != 4 || events[2].Phase != "rollback" || events[2].Outcome != "running" || events[3].Phase != "rollback" || events[3].Outcome != "succeeded" || events[2].OperationID != events[3].OperationID || events[2].ArtifactDigest != strings.Repeat("c", 64) {
		t.Fatalf("rollback events=%+v", events)
	}
	if len(operator.probes) != 4 {
		t.Fatalf("rollback probes=%#v", operator.probes)
	}
	for _, probe := range operator.probes {
		if !strings.HasSuffix(probe, "/healthz") && !strings.HasSuffix(probe, "/readyz") {
			t.Fatalf("rollback applied future-release smoke checks: %#v", operator.probes)
		}
	}
}

func TestDeploymentEvidenceCanNeverBlockActivationOrRollback(t *testing.T) {
	cfg, old, fresh := baseConfig(t, "singleton_candidate")
	cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
	m := manager(&fakeOperator{active: map[string]bool{}}, fresh)
	appendCalls := 0
	m.AppendEvent = func(string, eventlog.Event) error {
		appendCalls++
		return errors.New("injected event failure")
	}
	report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
	if err != nil {
		t.Fatal(err)
	}
	if report.EventWarnings != 2 || report.ActiveRelease != fresh {
		t.Fatalf("report=%+v", report)
	}
	if _, err := m.Rollback(context.Background(), cfg); err != nil {
		t.Fatal(err)
	}
	record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
	if err != nil || record.ActiveRelease != old || appendCalls != 4 {
		t.Fatalf("record=%+v appends=%d err=%v", record, appendCalls, err)
	}
}

func TestDeploymentEvidenceIdentityIsBestEffort(t *testing.T) {
	cfg, old, fresh := baseConfig(t, "singleton_candidate")
	cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
	m := manager(&fakeOperator{active: map[string]bool{}}, fresh)
	m.ReadIdentity = func(string) (releaseIdentity, error) {
		return releaseIdentity{}, errors.New("injected identity failure")
	}
	report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
	if err != nil || report.EventWarnings != 1 || report.ActiveRelease != fresh {
		t.Fatalf("report=%+v err=%v", report, err)
	}
}

func TestSingletonOperationIdentityIsRequiredBeforeCandidateStart(t *testing.T) {
	cfg, old, fresh := baseConfig(t, "singleton_candidate")
	cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
	operator := &fakeOperator{active: map[string]bool{}}
	m := manager(operator, fresh)
	m.OperationID = func() (string, error) { return "", errors.New("injected entropy failure") }
	if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err == nil || !strings.Contains(err.Error(), "operation identity") {
		t.Fatalf("expected operation identity refusal, got %v", err)
	}
	if len(operator.starts) != 0 {
		t.Fatalf("candidate started without an operation identity: %#v", operator.starts)
	}
}

func TestSingletonUnresolvedLeaseBlocksReplacementBeforeCandidateStart(t *testing.T) {
	cfg, old, fresh := baseConfig(t, "singleton_candidate")
	cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
	at := time.Unix(100, 0).UTC().Format(time.RFC3339)
	record := state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: fresh, CandidateRelease: fresh, ActiveSlot: "singleton", ActiveRelease: old, LastAttemptRelease: fresh, LastAttemptOutcome: "failed", LastAttemptAt: at, UpdatedAt: at}
	if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil {
		t.Fatal(err)
	}
	storeTestCandidateLease(t, cfg, fresh, at)
	operator := &fakeOperator{active: map[string]bool{testCandidateUnit: true}}
	if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err == nil || !strings.Contains(err.Error(), "run tend reconcile") {
		t.Fatalf("expected unresolved lease refusal, got %v", err)
	}
	if len(operator.starts) != 0 || !operator.active[testCandidateUnit] {
		t.Fatalf("existing candidate was disturbed: starts=%#v active=%#v", operator.starts, operator.active)
	}
}

func TestBlueGreenCaddyFailureRestoresHandlerAndSlot(t *testing.T) {
	cfg, old, fresh := baseConfig(t, "blue_green")
	handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
	template := filepath.Join(cfg.Deployment.Root, "handler.template")
	original := []byte("reverse_proxy 127.0.0.1:8090\n")
	_ = os.WriteFile(handler, original, 0o644)
	_ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644)
	blue := filepath.Join(cfg.Deployment.Root, "slots", "blue")
	green := filepath.Join(cfg.Deployment.Root, "slots", "green")
	_ = replaceSymlink(blue, old)
	_ = replaceSymlink(green, old)
	cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}}
	operator := &fakeOperator{active: map[string]bool{}, failReload: true}
	m := manager(operator, fresh)
	if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil {
		t.Fatal("expected failure")
	}
	body, _ := os.ReadFile(handler)
	if string(body) != string(original) {
		t.Fatalf("handler not restored: %q", body)
	}
	target, err := resolveReleaseLink(cfg.Deployment.Root, green)
	if err != nil || target != old {
		t.Fatalf("green=%q err=%v", target, err)
	}
	failed, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
	if err != nil {
		t.Fatal(err)
	}
	if failed.ActiveRelease != old || failed.LastAttemptOutcome != "failed" || failed.CandidateRelease != "" {
		t.Fatalf("failed state=%+v", failed)
	}
}

func TestSingletonRestartFailureRestoresPointers(t *testing.T) {
	cfg, old, fresh := baseConfig(t, "singleton_candidate")
	settings, handler := singletonSettings(t, cfg, old)
	cfg.Deployment.Singleton = settings
	operator := &fakeOperator{active: map[string]bool{}, failRestartUnit: "example-site.service", failRestartOnce: true}
	m := manager(operator, fresh)
	if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil {
		t.Fatal("expected failure")
	}
	target, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink)
	if err != nil || target != old {
		t.Fatalf("current=%q err=%v", target, err)
	}
	if _, err := os.Lstat(cfg.Deployment.Singleton.PreviousLink); !os.IsNotExist(err) {
		t.Fatal("previous pointer was not restored")
	}
	body, err := os.ReadFile(handler)
	if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
		t.Fatalf("handler=%q err=%v", body, err)
	}
	if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 {
		t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err)
	}
	if operator.active[testCandidateUnit] {
		t.Fatal("candidate was not stopped after successful restoration")
	}
}

func TestStateRecordsSuccessfulActivation(t *testing.T) {
	cfg, old, fresh := baseConfig(t, "singleton_candidate")
	settings, handler := singletonSettings(t, cfg, old)
	cfg.Deployment.Singleton = settings
	operator := &fakeOperator{active: map[string]bool{}}
	m := manager(operator, fresh)
	if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err != nil {
		t.Fatal(err)
	}
	record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
	if err != nil {
		t.Fatal(err)
	}
	if record.ActiveRelease != fresh || record.PreviousRelease != old || record.CandidateRelease != "" {
		t.Fatalf("state=%+v", record)
	}
	if _, err := os.Lstat(state.CandidateLeasePath(cfg.Deployment.StateFile)); !os.IsNotExist(err) {
		t.Fatalf("candidate lease was not removed: %v", err)
	}
	if len(operator.starts) != 1 || operator.starts[0] != testCandidateUnit {
		t.Fatalf("candidate starts=%#v", operator.starts)
	}
	if operator.candidateFile != cfg.Service.EnvironmentFile || len(operator.candidateEnvironment) != 1 || operator.candidateEnvironment["EXAMPLE_LISTEN"] != "127.0.0.1:18092" {
		t.Fatalf("candidate file=%q environment=%#v", operator.candidateFile, operator.candidateEnvironment)
	}
	body, err := os.ReadFile(handler)
	if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
		t.Fatalf("handler=%q err=%v", body, err)
	}
	if operator.reloads != 2 || operator.active[testCandidateUnit] {
		t.Fatalf("reloads=%d active=%#v", operator.reloads, operator.active)
	}
	reconciliation, err := m.Reconcile(context.Background(), cfg)
	if err != nil || reconciliation.Mutation != "none" || reconciliation.Disposition != "settled" || reconciliation.Observed.CandidateLease || !reconciliation.Consistent {
		t.Fatalf("reconciliation=%+v err=%v", reconciliation, err)
	}
}

func TestSingletonContinuityFailureRestoresHandlerPointersAndService(t *testing.T) {
	cfg, old, fresh := baseConfig(t, "singleton_candidate")
	settings, handler := singletonSettings(t, cfg, old)
	cfg.Deployment.Singleton = settings
	operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failPublicAfter: 4}
	m := manager(operator, fresh)
	if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil || !strings.Contains(err.Error(), "continuity") {
		t.Fatalf("expected continuity failure, got %v", err)
	}
	current, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink)
	if err != nil || current != old {
		t.Fatalf("current=%q err=%v", current, err)
	}
	body, err := os.ReadFile(handler)
	if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
		t.Fatalf("handler=%q err=%v", body, err)
	}
	if operator.active[testCandidateUnit] {
		t.Fatal("candidate was not stopped after continuity restoration")
	}
}

func TestSingletonCaddyReloadFailuresRestorePriorRoute(t *testing.T) {
	for _, reload := range []int{1, 2} {
		t.Run(fmt.Sprintf("reload-%d", reload), func(t *testing.T) {
			cfg, old, fresh := baseConfig(t, "singleton_candidate")
			settings, handler := singletonSettings(t, cfg, old)
			cfg.Deployment.Singleton = settings
			operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failReloadAt: reload}
			if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true}); err == nil || !strings.Contains(err.Error(), "Caddy reload failed") {
				t.Fatalf("expected Caddy reload failure, got %v", err)
			}
			current, err := resolveReleaseLink(cfg.Deployment.Root, settings.CurrentLink)
			if err != nil || current != old {
				t.Fatalf("current=%q err=%v", current, err)
			}
			body, err := os.ReadFile(handler)
			if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
				t.Fatalf("handler=%q err=%v", body, err)
			}
			if operator.active[testCandidateUnit] {
				t.Fatal("candidate was not stopped after route restoration")
			}
		})
	}
}

func TestSingletonIncompleteRecoveryKeepsProvenCandidateRunning(t *testing.T) {
	cfg, old, fresh := baseConfig(t, "singleton_candidate")
	settings, _ := singletonSettings(t, cfg, old)
	cfg.Deployment.Singleton = settings
	operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failReload: true}
	_, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true})
	if err == nil || !strings.Contains(err.Error(), "candidate remains routed for operator recovery") {
		t.Fatalf("expected explicit incomplete recovery, got %v", err)
	}
	if !operator.active[testCandidateUnit] {
		t.Fatal("proven candidate was stopped despite incomplete route restoration")
	}
	record, loadErr := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
	lease, leaseErr := state.LoadCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, cfg.Service.Name)
	if loadErr != nil || leaseErr != nil || record.CandidateRelease != fresh || lease.OperationID != strings.Repeat("d", 32) || lease.Unit != testCandidateUnit {
		t.Fatalf("retained state=%+v err=%v", record, loadErr)
	}
}

func TestReconcileReportsRetainedCandidateWithoutMutation(t *testing.T) {
	cfg, old, fresh := baseConfig(t, "singleton_candidate")
	settings, handler := singletonSettings(t, cfg, old)
	cfg.Deployment.Singleton = settings
	at := time.Unix(100, 0).UTC().Format(time.RFC3339)
	record := state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: fresh, CandidateRelease: fresh, ActiveSlot: "singleton", ActiveRelease: old, LastAttemptRelease: fresh, LastAttemptOutcome: "failed", LastAttemptAt: at, UpdatedAt: at}
	if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil {
		t.Fatal(err)
	}
	storeTestCandidateLease(t, cfg, fresh, at)
	candidateHandler, err := renderHandler(settings.CaddyHandlerTemplate, settings.CandidateAddress)
	if err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(handler, candidateHandler, 0o640); err != nil {
		t.Fatal(err)
	}
	operator := &fakeOperator{active: map[string]bool{settings.Unit: true, testCandidateUnit: true}}
	reconciliation, err := manager(operator, fresh).Reconcile(context.Background(), cfg)
	if err != nil || reconciliation.Mutation != "none" || !reconciliation.Observed.CandidateLease || reconciliation.Observed.CandidateUnitActive == nil || !*reconciliation.Observed.CandidateUnitActive || reconciliation.Observed.HandlerFileTarget != "candidate" || reconciliation.Disposition != "retained_candidate_handler_file" {
		t.Fatalf("reconciliation=%+v err=%v", reconciliation, err)
	}
	if len(operator.stops) != 0 || len(operator.restarts) != 0 || operator.reloads != 0 {
		t.Fatalf("reconcile mutated services: stops=%#v restarts=%#v reloads=%d", operator.stops, operator.restarts, operator.reloads)
	}
}

func TestSingletonRollbackRetainsOperationLeaseWhenRecoveryIsIncomplete(t *testing.T) {
	cfg, old, fresh := baseConfig(t, "singleton_candidate")
	cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
	operator := &fakeOperator{active: map[string]bool{cfg.Deployment.Singleton.Unit: true}}
	m := manager(operator, fresh)
	if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err != nil {
		t.Fatal(err)
	}
	operator.failReload = true
	if _, err := m.Rollback(context.Background(), cfg); err == nil || !strings.Contains(err.Error(), "candidate remains routed for operator recovery") {
		t.Fatalf("expected retained rollback candidate, got %v", err)
	}
	record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
	if err != nil {
		t.Fatal(err)
	}
	lease, leaseErr := state.LoadCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, cfg.Service.Name)
	if record.ActiveRelease != fresh || record.CandidateRelease != old || leaseErr != nil || lease.OperationID != strings.Repeat("d", 32) || lease.Unit != testCandidateUnit || record.LastAttemptOutcome != "failed" {
		t.Fatalf("rollback state=%+v", record)
	}
	if !operator.active[testCandidateUnit] {
		t.Fatal("rollback candidate was stopped despite incomplete recovery")
	}
}

func TestPruneProtectsOperationScopedCandidateRelease(t *testing.T) {
	cfg, active, candidate := baseConfig(t, "singleton_candidate")
	cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, active)
	at := time.Unix(100, 0).UTC().Format(time.RFC3339)
	record := state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: candidate, CandidateRelease: candidate, ActiveSlot: "singleton", ActiveRelease: active, LastAttemptRelease: candidate, LastAttemptOutcome: "failed", LastAttemptAt: at, UpdatedAt: at}
	if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil {
		t.Fatal(err)
	}
	storeTestCandidateLease(t, cfg, candidate, at)
	removed, err := manager(&fakeOperator{active: map[string]bool{}}, candidate).Prune(cfg, 2, true)
	if err != nil {
		t.Fatal(err)
	}
	if len(removed) != 0 {
		t.Fatalf("candidate release was selected for pruning: %#v", removed)
	}
	if info, err := os.Stat(candidate); err != nil || !info.IsDir() {
		t.Fatalf("candidate release was not preserved: %v", err)
	}
}