docs: publish Tend Compose continuity evidence
Export the reviewed allowlisted snapshot from private source commit 07c1655921f21ee5e4fc4d85639d199e8867b17d. This records the Docker Compose activation, schema-compatible rollback, and stateful migration resource findings from Observatory Preview 19 dogfooding. AI-Assisted: OpenAI Codex Signed-off-by: Cole Speelman <crspeelman@gmail.com>
This commit is contained in:
@@ -0,0 +1,591 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/tend/internal/config"
|
||||
"gamertan.com/tend/internal/eventlog"
|
||||
"gamertan.com/tend/internal/state"
|
||||
)
|
||||
|
||||
const testCandidateUnit = "example-site-tend-candidate-dddddddddddd.service"
|
||||
|
||||
func storeTestCandidateLease(t *testing.T, cfg config.Config, release, startedAt string) {
|
||||
t.Helper()
|
||||
lease := state.CandidateLease{SchemaVersion: state.CandidateLeaseSchemaVersion, Service: cfg.Service.Name, OperationID: strings.Repeat("d", 32), Release: release, Unit: testCandidateUnit, Address: cfg.Deployment.Singleton.CandidateAddress, StartedAt: startedAt}
|
||||
if err := state.StoreCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, lease); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeOperator struct {
|
||||
failReload bool
|
||||
failReloadAt int
|
||||
reloads int
|
||||
failRestartUnit string
|
||||
failRestartOnce bool
|
||||
rejectMarkers bool
|
||||
active map[string]bool
|
||||
starts, stops, restarts []string
|
||||
probes []string
|
||||
publicProbes []string
|
||||
failPublic bool
|
||||
failPublicAfter int
|
||||
candidateEnvironment map[string]string
|
||||
candidateFile string
|
||||
}
|
||||
|
||||
func (f *fakeOperator) Restart(_ context.Context, unit string) error {
|
||||
f.restarts = append(f.restarts, unit)
|
||||
if unit == f.failRestartUnit {
|
||||
if f.failRestartOnce {
|
||||
f.failRestartUnit = ""
|
||||
}
|
||||
return errors.New("injected restart failure")
|
||||
}
|
||||
f.active[unit] = true
|
||||
return nil
|
||||
}
|
||||
func (f *fakeOperator) Stop(_ context.Context, unit string) error {
|
||||
f.stops = append(f.stops, unit)
|
||||
f.active[unit] = false
|
||||
return nil
|
||||
}
|
||||
func (f *fakeOperator) IsActive(_ context.Context, unit string) (bool, error) {
|
||||
return f.active[unit], nil
|
||||
}
|
||||
func (f *fakeOperator) StartCandidate(_ context.Context, unit, binary, environmentFile string, env map[string]string) error {
|
||||
if !filepath.IsAbs(binary) || !filepath.IsAbs(environmentFile) || len(env) == 0 {
|
||||
return errors.New("bad candidate")
|
||||
}
|
||||
f.starts = append(f.starts, unit)
|
||||
f.candidateFile = environmentFile
|
||||
f.candidateEnvironment = make(map[string]string, len(env))
|
||||
for key, value := range env {
|
||||
f.candidateEnvironment[key] = value
|
||||
}
|
||||
f.active[unit] = true
|
||||
return nil
|
||||
}
|
||||
func (f *fakeOperator) ProbeURL(_ context.Context, value, contains string) error {
|
||||
f.publicProbes = append(f.publicProbes, value)
|
||||
if f.failPublic || (f.failPublicAfter > 0 && len(f.publicProbes) >= f.failPublicAfter) {
|
||||
return errors.New("injected public smoke failure")
|
||||
}
|
||||
if f.rejectMarkers && contains != "" {
|
||||
return errors.New("unexpected future-release smoke marker")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestPublicSmokeFailureRestoresBlueGreenHandlerAndSlot(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "blue_green")
|
||||
handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
|
||||
template := filepath.Join(cfg.Deployment.Root, "handler.template")
|
||||
original := []byte("reverse_proxy 127.0.0.1:8090\n")
|
||||
_ = os.WriteFile(handler, original, 0o644)
|
||||
_ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644)
|
||||
blue := filepath.Join(cfg.Deployment.Root, "slots", "blue")
|
||||
green := filepath.Join(cfg.Deployment.Root, "slots", "green")
|
||||
_ = replaceSymlink(blue, old)
|
||||
_ = replaceSymlink(green, old)
|
||||
cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}}
|
||||
operator := &fakeOperator{active: map[string]bool{}, failPublic: true}
|
||||
if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true}); err == nil {
|
||||
t.Fatal("expected public smoke failure")
|
||||
}
|
||||
body, _ := os.ReadFile(handler)
|
||||
if string(body) != string(original) {
|
||||
t.Fatalf("handler not restored: %q", body)
|
||||
}
|
||||
target, err := resolveReleaseLink(cfg.Deployment.Root, green)
|
||||
if err != nil || target != old {
|
||||
t.Fatalf("green=%q err=%v", target, err)
|
||||
}
|
||||
record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if record.ActiveRelease != old || record.LastAttemptOutcome != "failed" || record.CandidateRelease != "" || record.LastAttemptRelease != fresh {
|
||||
t.Fatalf("failed attempt state=%+v", record)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContinuityFailureRestoresBlueGreenRoute(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "blue_green")
|
||||
handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
|
||||
template := filepath.Join(cfg.Deployment.Root, "handler.template")
|
||||
original := []byte("reverse_proxy 127.0.0.1:8090\n")
|
||||
_ = os.WriteFile(handler, original, 0o644)
|
||||
_ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644)
|
||||
blue := filepath.Join(cfg.Deployment.Root, "slots", "blue")
|
||||
green := filepath.Join(cfg.Deployment.Root, "slots", "green")
|
||||
_ = replaceSymlink(blue, old)
|
||||
_ = replaceSymlink(green, old)
|
||||
cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}}
|
||||
operator := &fakeOperator{active: map[string]bool{}, failPublicAfter: 3}
|
||||
if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err == nil || !strings.Contains(err.Error(), "continuity") {
|
||||
t.Fatalf("expected continuity failure, got %v", err)
|
||||
}
|
||||
body, _ := os.ReadFile(handler)
|
||||
if string(body) != string(original) {
|
||||
t.Fatalf("handler not restored: %q", body)
|
||||
}
|
||||
target, err := resolveReleaseLink(cfg.Deployment.Root, green)
|
||||
if err != nil || target != old {
|
||||
t.Fatalf("green=%q err=%v", target, err)
|
||||
}
|
||||
}
|
||||
func (f *fakeOperator) ValidateCaddy(context.Context, string) error { return nil }
|
||||
func (f *fakeOperator) ReloadCaddy(context.Context) error {
|
||||
f.reloads++
|
||||
if f.failReload || (f.failReloadAt > 0 && f.reloads == f.failReloadAt) {
|
||||
return errors.New("injected reload failure")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (f *fakeOperator) Probe(_ context.Context, address, host, path, contains string) error {
|
||||
f.probes = append(f.probes, address+path)
|
||||
if f.rejectMarkers && contains != "" {
|
||||
return errors.New("unexpected future-release smoke marker")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func baseConfig(t *testing.T, strategy string) (config.Config, string, string) {
|
||||
t.Helper()
|
||||
root := filepath.Join(t.TempDir(), "service")
|
||||
if err := os.MkdirAll(filepath.Join(root, "releases"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := filepath.Join(root, "releases", "sha256-"+strings.Repeat("c", 64))
|
||||
fresh := filepath.Join(root, "releases", "sha256-"+strings.Repeat("a", 64))
|
||||
for _, dir := range []string{old, fresh} {
|
||||
if err := os.Mkdir(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "app"), []byte("x"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
cfg := config.Config{SchemaVersion: 2, Service: config.Service{Name: "example-site", AllowedHost: "example.test", EnvironmentFile: "/etc/tend/environment/example-site.env"}, Build: config.Build{Package: "./cmd/site", Binary: "app", Branch: "main"}, Deployment: config.Deployment{Strategy: strategy, Root: root, LockFile: filepath.Join(root, "deploy.lock"), StateFile: filepath.Join(root, "state.json"), EventLog: filepath.Join(root, "deployment-events.jsonl"), HealthPath: "/healthz", ReadinessPath: "/readyz", CandidateTimeoutSecs: 2, ActivationWindowSecs: 1, Smoke: []config.Smoke{{Path: "/", Contains: "Example"}}, PublicSmoke: []config.PublicSmoke{{URL: "https://example.test/", Contains: "Example"}}}}
|
||||
return cfg, old, fresh
|
||||
}
|
||||
func manager(operator Operator, fresh string) Manager {
|
||||
return Manager{Operator: operator, Now: func() time.Time { return time.Unix(100, 0).UTC() }, Prepare: func(config.Config, string, string, string) (string, error) { return fresh, nil }, Inspect: func(config.Config, string, string, string) error { return nil }, ReadIdentity: func(string) (releaseIdentity, error) {
|
||||
return releaseIdentity{Version: "v0.2.0-preview.1", Commit: strings.Repeat("b", 40)}, nil
|
||||
}, OperationID: func() (string, error) { return strings.Repeat("d", 32), nil }, Sleep: func(context.Context, time.Duration) error { return nil }}
|
||||
}
|
||||
|
||||
func singletonSettings(t *testing.T, cfg config.Config, currentRelease string) (*config.Singleton, string) {
|
||||
t.Helper()
|
||||
handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
|
||||
template := filepath.Join(cfg.Deployment.Root, "handler.template")
|
||||
if err := os.WriteFile(handler, []byte("reverse_proxy 127.0.0.1:8092\n"), 0o640); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current := filepath.Join(cfg.Deployment.Root, "current")
|
||||
if err := replaceSymlink(current, currentRelease); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &config.Singleton{
|
||||
Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN",
|
||||
CurrentLink: current, PreviousLink: filepath.Join(cfg.Deployment.Root, "previous"), CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"),
|
||||
CaddyHandler: handler, CaddyHandlerTemplate: template,
|
||||
}, handler
|
||||
}
|
||||
|
||||
func TestBlueGreenActivationAndRollback(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "blue_green")
|
||||
handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
|
||||
template := filepath.Join(cfg.Deployment.Root, "handler.template")
|
||||
if err := os.WriteFile(handler, []byte("reverse_proxy 127.0.0.1:8090\n"), 0o640); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
blue := filepath.Join(cfg.Deployment.Root, "slots", "blue")
|
||||
green := filepath.Join(cfg.Deployment.Root, "slots", "green")
|
||||
if err := replaceSymlink(blue, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := replaceSymlink(green, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
operator := &fakeOperator{active: map[string]bool{"example-blue.service": true, "example-green.service": true}}
|
||||
m := manager(operator, fresh)
|
||||
var events []eventlog.Event
|
||||
m.AppendEvent = func(_ string, event eventlog.Event) error {
|
||||
events = append(events, event)
|
||||
return nil
|
||||
}
|
||||
report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.ActiveRelease != fresh || report.PreviousRelease != old {
|
||||
t.Fatalf("report=%+v", report)
|
||||
}
|
||||
deployed, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if deployed.DesiredRelease != fresh || deployed.CandidateRelease != "" || deployed.LastAttemptRelease != fresh || deployed.LastAttemptOutcome != "succeeded" {
|
||||
t.Fatalf("deployment identity state=%+v", deployed)
|
||||
}
|
||||
if len(events) != 2 || events[0].Phase != "candidate" || events[0].Outcome != "running" || events[1].Phase != "activation" || events[1].Outcome != "succeeded" || events[0].OperationID != events[1].OperationID {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 {
|
||||
t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err)
|
||||
}
|
||||
if target, err := resolveReleaseLink(cfg.Deployment.Root, green); err != nil || target != fresh {
|
||||
t.Fatalf("green=%q err=%v", target, err)
|
||||
}
|
||||
operator.rejectMarkers = true
|
||||
operator.probes = nil
|
||||
record, err := m.Rollback(context.Background(), cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if record.ActiveRelease != old || record.PreviousRelease != fresh {
|
||||
t.Fatalf("rollback=%+v", record)
|
||||
}
|
||||
if len(events) != 4 || events[2].Phase != "rollback" || events[2].Outcome != "running" || events[3].Phase != "rollback" || events[3].Outcome != "succeeded" || events[2].OperationID != events[3].OperationID || events[2].ArtifactDigest != strings.Repeat("c", 64) {
|
||||
t.Fatalf("rollback events=%+v", events)
|
||||
}
|
||||
if len(operator.probes) != 4 {
|
||||
t.Fatalf("rollback probes=%#v", operator.probes)
|
||||
}
|
||||
for _, probe := range operator.probes {
|
||||
if !strings.HasSuffix(probe, "/healthz") && !strings.HasSuffix(probe, "/readyz") {
|
||||
t.Fatalf("rollback applied future-release smoke checks: %#v", operator.probes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeploymentEvidenceCanNeverBlockActivationOrRollback(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
|
||||
m := manager(&fakeOperator{active: map[string]bool{}}, fresh)
|
||||
appendCalls := 0
|
||||
m.AppendEvent = func(string, eventlog.Event) error {
|
||||
appendCalls++
|
||||
return errors.New("injected event failure")
|
||||
}
|
||||
report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.EventWarnings != 2 || report.ActiveRelease != fresh {
|
||||
t.Fatalf("report=%+v", report)
|
||||
}
|
||||
if _, err := m.Rollback(context.Background(), cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err != nil || record.ActiveRelease != old || appendCalls != 4 {
|
||||
t.Fatalf("record=%+v appends=%d err=%v", record, appendCalls, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeploymentEvidenceIdentityIsBestEffort(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
|
||||
m := manager(&fakeOperator{active: map[string]bool{}}, fresh)
|
||||
m.ReadIdentity = func(string) (releaseIdentity, error) {
|
||||
return releaseIdentity{}, errors.New("injected identity failure")
|
||||
}
|
||||
report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
|
||||
if err != nil || report.EventWarnings != 1 || report.ActiveRelease != fresh {
|
||||
t.Fatalf("report=%+v err=%v", report, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingletonOperationIdentityIsRequiredBeforeCandidateStart(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
|
||||
operator := &fakeOperator{active: map[string]bool{}}
|
||||
m := manager(operator, fresh)
|
||||
m.OperationID = func() (string, error) { return "", errors.New("injected entropy failure") }
|
||||
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err == nil || !strings.Contains(err.Error(), "operation identity") {
|
||||
t.Fatalf("expected operation identity refusal, got %v", err)
|
||||
}
|
||||
if len(operator.starts) != 0 {
|
||||
t.Fatalf("candidate started without an operation identity: %#v", operator.starts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingletonUnresolvedLeaseBlocksReplacementBeforeCandidateStart(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
|
||||
at := time.Unix(100, 0).UTC().Format(time.RFC3339)
|
||||
record := state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: fresh, CandidateRelease: fresh, ActiveSlot: "singleton", ActiveRelease: old, LastAttemptRelease: fresh, LastAttemptOutcome: "failed", LastAttemptAt: at, UpdatedAt: at}
|
||||
if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
storeTestCandidateLease(t, cfg, fresh, at)
|
||||
operator := &fakeOperator{active: map[string]bool{testCandidateUnit: true}}
|
||||
if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err == nil || !strings.Contains(err.Error(), "run tend reconcile") {
|
||||
t.Fatalf("expected unresolved lease refusal, got %v", err)
|
||||
}
|
||||
if len(operator.starts) != 0 || !operator.active[testCandidateUnit] {
|
||||
t.Fatalf("existing candidate was disturbed: starts=%#v active=%#v", operator.starts, operator.active)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlueGreenCaddyFailureRestoresHandlerAndSlot(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "blue_green")
|
||||
handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
|
||||
template := filepath.Join(cfg.Deployment.Root, "handler.template")
|
||||
original := []byte("reverse_proxy 127.0.0.1:8090\n")
|
||||
_ = os.WriteFile(handler, original, 0o644)
|
||||
_ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644)
|
||||
blue := filepath.Join(cfg.Deployment.Root, "slots", "blue")
|
||||
green := filepath.Join(cfg.Deployment.Root, "slots", "green")
|
||||
_ = replaceSymlink(blue, old)
|
||||
_ = replaceSymlink(green, old)
|
||||
cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}}
|
||||
operator := &fakeOperator{active: map[string]bool{}, failReload: true}
|
||||
m := manager(operator, fresh)
|
||||
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil {
|
||||
t.Fatal("expected failure")
|
||||
}
|
||||
body, _ := os.ReadFile(handler)
|
||||
if string(body) != string(original) {
|
||||
t.Fatalf("handler not restored: %q", body)
|
||||
}
|
||||
target, err := resolveReleaseLink(cfg.Deployment.Root, green)
|
||||
if err != nil || target != old {
|
||||
t.Fatalf("green=%q err=%v", target, err)
|
||||
}
|
||||
failed, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if failed.ActiveRelease != old || failed.LastAttemptOutcome != "failed" || failed.CandidateRelease != "" {
|
||||
t.Fatalf("failed state=%+v", failed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingletonRestartFailureRestoresPointers(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
settings, handler := singletonSettings(t, cfg, old)
|
||||
cfg.Deployment.Singleton = settings
|
||||
operator := &fakeOperator{active: map[string]bool{}, failRestartUnit: "example-site.service", failRestartOnce: true}
|
||||
m := manager(operator, fresh)
|
||||
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil {
|
||||
t.Fatal("expected failure")
|
||||
}
|
||||
target, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink)
|
||||
if err != nil || target != old {
|
||||
t.Fatalf("current=%q err=%v", target, err)
|
||||
}
|
||||
if _, err := os.Lstat(cfg.Deployment.Singleton.PreviousLink); !os.IsNotExist(err) {
|
||||
t.Fatal("previous pointer was not restored")
|
||||
}
|
||||
body, err := os.ReadFile(handler)
|
||||
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
|
||||
t.Fatalf("handler=%q err=%v", body, err)
|
||||
}
|
||||
if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 {
|
||||
t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err)
|
||||
}
|
||||
if operator.active[testCandidateUnit] {
|
||||
t.Fatal("candidate was not stopped after successful restoration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStateRecordsSuccessfulActivation(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
settings, handler := singletonSettings(t, cfg, old)
|
||||
cfg.Deployment.Singleton = settings
|
||||
operator := &fakeOperator{active: map[string]bool{}}
|
||||
m := manager(operator, fresh)
|
||||
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if record.ActiveRelease != fresh || record.PreviousRelease != old || record.CandidateRelease != "" {
|
||||
t.Fatalf("state=%+v", record)
|
||||
}
|
||||
if _, err := os.Lstat(state.CandidateLeasePath(cfg.Deployment.StateFile)); !os.IsNotExist(err) {
|
||||
t.Fatalf("candidate lease was not removed: %v", err)
|
||||
}
|
||||
if len(operator.starts) != 1 || operator.starts[0] != testCandidateUnit {
|
||||
t.Fatalf("candidate starts=%#v", operator.starts)
|
||||
}
|
||||
if operator.candidateFile != cfg.Service.EnvironmentFile || len(operator.candidateEnvironment) != 1 || operator.candidateEnvironment["EXAMPLE_LISTEN"] != "127.0.0.1:18092" {
|
||||
t.Fatalf("candidate file=%q environment=%#v", operator.candidateFile, operator.candidateEnvironment)
|
||||
}
|
||||
body, err := os.ReadFile(handler)
|
||||
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
|
||||
t.Fatalf("handler=%q err=%v", body, err)
|
||||
}
|
||||
if operator.reloads != 2 || operator.active[testCandidateUnit] {
|
||||
t.Fatalf("reloads=%d active=%#v", operator.reloads, operator.active)
|
||||
}
|
||||
reconciliation, err := m.Reconcile(context.Background(), cfg)
|
||||
if err != nil || reconciliation.Mutation != "none" || reconciliation.Disposition != "settled" || reconciliation.Observed.CandidateLease || !reconciliation.Consistent {
|
||||
t.Fatalf("reconciliation=%+v err=%v", reconciliation, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingletonContinuityFailureRestoresHandlerPointersAndService(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
settings, handler := singletonSettings(t, cfg, old)
|
||||
cfg.Deployment.Singleton = settings
|
||||
operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failPublicAfter: 4}
|
||||
m := manager(operator, fresh)
|
||||
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil || !strings.Contains(err.Error(), "continuity") {
|
||||
t.Fatalf("expected continuity failure, got %v", err)
|
||||
}
|
||||
current, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink)
|
||||
if err != nil || current != old {
|
||||
t.Fatalf("current=%q err=%v", current, err)
|
||||
}
|
||||
body, err := os.ReadFile(handler)
|
||||
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
|
||||
t.Fatalf("handler=%q err=%v", body, err)
|
||||
}
|
||||
if operator.active[testCandidateUnit] {
|
||||
t.Fatal("candidate was not stopped after continuity restoration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingletonCaddyReloadFailuresRestorePriorRoute(t *testing.T) {
|
||||
for _, reload := range []int{1, 2} {
|
||||
t.Run(fmt.Sprintf("reload-%d", reload), func(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
settings, handler := singletonSettings(t, cfg, old)
|
||||
cfg.Deployment.Singleton = settings
|
||||
operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failReloadAt: reload}
|
||||
if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true}); err == nil || !strings.Contains(err.Error(), "Caddy reload failed") {
|
||||
t.Fatalf("expected Caddy reload failure, got %v", err)
|
||||
}
|
||||
current, err := resolveReleaseLink(cfg.Deployment.Root, settings.CurrentLink)
|
||||
if err != nil || current != old {
|
||||
t.Fatalf("current=%q err=%v", current, err)
|
||||
}
|
||||
body, err := os.ReadFile(handler)
|
||||
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
|
||||
t.Fatalf("handler=%q err=%v", body, err)
|
||||
}
|
||||
if operator.active[testCandidateUnit] {
|
||||
t.Fatal("candidate was not stopped after route restoration")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingletonIncompleteRecoveryKeepsProvenCandidateRunning(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
settings, _ := singletonSettings(t, cfg, old)
|
||||
cfg.Deployment.Singleton = settings
|
||||
operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failReload: true}
|
||||
_, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true})
|
||||
if err == nil || !strings.Contains(err.Error(), "candidate remains routed for operator recovery") {
|
||||
t.Fatalf("expected explicit incomplete recovery, got %v", err)
|
||||
}
|
||||
if !operator.active[testCandidateUnit] {
|
||||
t.Fatal("proven candidate was stopped despite incomplete route restoration")
|
||||
}
|
||||
record, loadErr := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
lease, leaseErr := state.LoadCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, cfg.Service.Name)
|
||||
if loadErr != nil || leaseErr != nil || record.CandidateRelease != fresh || lease.OperationID != strings.Repeat("d", 32) || lease.Unit != testCandidateUnit {
|
||||
t.Fatalf("retained state=%+v err=%v", record, loadErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReconcileReportsRetainedCandidateWithoutMutation(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
settings, handler := singletonSettings(t, cfg, old)
|
||||
cfg.Deployment.Singleton = settings
|
||||
at := time.Unix(100, 0).UTC().Format(time.RFC3339)
|
||||
record := state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: fresh, CandidateRelease: fresh, ActiveSlot: "singleton", ActiveRelease: old, LastAttemptRelease: fresh, LastAttemptOutcome: "failed", LastAttemptAt: at, UpdatedAt: at}
|
||||
if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
storeTestCandidateLease(t, cfg, fresh, at)
|
||||
candidateHandler, err := renderHandler(settings.CaddyHandlerTemplate, settings.CandidateAddress)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(handler, candidateHandler, 0o640); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
operator := &fakeOperator{active: map[string]bool{settings.Unit: true, testCandidateUnit: true}}
|
||||
reconciliation, err := manager(operator, fresh).Reconcile(context.Background(), cfg)
|
||||
if err != nil || reconciliation.Mutation != "none" || !reconciliation.Observed.CandidateLease || reconciliation.Observed.CandidateUnitActive == nil || !*reconciliation.Observed.CandidateUnitActive || reconciliation.Observed.HandlerFileTarget != "candidate" || reconciliation.Disposition != "retained_candidate_handler_file" {
|
||||
t.Fatalf("reconciliation=%+v err=%v", reconciliation, err)
|
||||
}
|
||||
if len(operator.stops) != 0 || len(operator.restarts) != 0 || operator.reloads != 0 {
|
||||
t.Fatalf("reconcile mutated services: stops=%#v restarts=%#v reloads=%d", operator.stops, operator.restarts, operator.reloads)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingletonRollbackRetainsOperationLeaseWhenRecoveryIsIncomplete(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
|
||||
operator := &fakeOperator{active: map[string]bool{cfg.Deployment.Singleton.Unit: true}}
|
||||
m := manager(operator, fresh)
|
||||
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
operator.failReload = true
|
||||
if _, err := m.Rollback(context.Background(), cfg); err == nil || !strings.Contains(err.Error(), "candidate remains routed for operator recovery") {
|
||||
t.Fatalf("expected retained rollback candidate, got %v", err)
|
||||
}
|
||||
record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
lease, leaseErr := state.LoadCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, cfg.Service.Name)
|
||||
if record.ActiveRelease != fresh || record.CandidateRelease != old || leaseErr != nil || lease.OperationID != strings.Repeat("d", 32) || lease.Unit != testCandidateUnit || record.LastAttemptOutcome != "failed" {
|
||||
t.Fatalf("rollback state=%+v", record)
|
||||
}
|
||||
if !operator.active[testCandidateUnit] {
|
||||
t.Fatal("rollback candidate was stopped despite incomplete recovery")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPruneProtectsOperationScopedCandidateRelease(t *testing.T) {
|
||||
cfg, active, candidate := baseConfig(t, "singleton_candidate")
|
||||
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, active)
|
||||
at := time.Unix(100, 0).UTC().Format(time.RFC3339)
|
||||
record := state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: candidate, CandidateRelease: candidate, ActiveSlot: "singleton", ActiveRelease: active, LastAttemptRelease: candidate, LastAttemptOutcome: "failed", LastAttemptAt: at, UpdatedAt: at}
|
||||
if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
storeTestCandidateLease(t, cfg, candidate, at)
|
||||
removed, err := manager(&fakeOperator{active: map[string]bool{}}, candidate).Prune(cfg, 2, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(removed) != 0 {
|
||||
t.Fatalf("candidate release was selected for pruning: %#v", removed)
|
||||
}
|
||||
if info, err := os.Stat(candidate); err != nil || !info.IsDir() {
|
||||
t.Fatalf("candidate release was not preserved: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user