feat: publish Tend v0.2 Preview 2 source

Export the reviewed allowlisted snapshot from private source commit 8aab3db43f35e6a49aa497f45d73701b13fc9f32 and tree 992132ea4703437dc13ffdbb04a077816c02caf9. This includes routed singleton continuity, deployment evidence, strict schema-2 configuration, restricted transport, and the independently compilable public-tree guard. AI-Assisted: OpenAI Codex Signed-off-by: Cole Speelman <crspeelman@gmail.com>
2026-08-18 06:40:58 -04:00
parent 00d1dd4209
commit 9d9fc83dd0
33 changed files with 1463 additions and 150 deletions
@@ -5,6 +5,7 @@ package deploy
 import (
 	"context"
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -12,18 +13,23 @@ import (
 	"time"

 	"gamertan.com/tend/internal/config"
+	"gamertan.com/tend/internal/eventlog"
 	"gamertan.com/tend/internal/state"
 )

 type fakeOperator struct {
 	failReload              bool
+	failReloadAt            int
+	reloads                 int
 	failRestartUnit         string
+	failRestartOnce         bool
 	rejectMarkers           bool
 	active                  map[string]bool
 	starts, stops, restarts []string
 	probes                  []string
 	publicProbes            []string
 	failPublic              bool
+	failPublicAfter         int
 	candidateEnvironment    map[string]string
 	candidateFile           string
 }
@@ -31,6 +37,9 @@ type fakeOperator struct {
 func (f *fakeOperator) Restart(_ context.Context, unit string) error {
 	f.restarts = append(f.restarts, unit)
 	if unit == f.failRestartUnit {
+		if f.failRestartOnce {
+			f.failRestartUnit = ""
+		}
 		return errors.New("injected restart failure")
 	}
 	f.active[unit] = true
@@ -59,7 +68,7 @@ func (f *fakeOperator) StartCandidate(_ context.Context, unit, binary, environme
 }
 func (f *fakeOperator) ProbeURL(_ context.Context, value, contains string) error {
 	f.publicProbes = append(f.publicProbes, value)
-	if f.failPublic {
+	if f.failPublic || (f.failPublicAfter > 0 && len(f.publicProbes) >= f.failPublicAfter) {
 		return errors.New("injected public smoke failure")
 	}
 	if f.rejectMarkers && contains != "" {
@@ -92,13 +101,44 @@ func TestPublicSmokeFailureRestoresBlueGreenHandlerAndSlot(t *testing.T) {
 	if err != nil || target != old {
 		t.Fatalf("green=%q err=%v", target, err)
 	}
-	if _, err = os.Stat(cfg.Deployment.StateFile); !os.IsNotExist(err) {
-		t.Fatal("failed public smoke wrote state")
+	record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if record.ActiveRelease != old || record.LastAttemptOutcome != "failed" || record.CandidateRelease != "" || record.LastAttemptRelease != fresh {
+		t.Fatalf("failed attempt state=%+v", record)
+	}
+}
+
+func TestContinuityFailureRestoresBlueGreenRoute(t *testing.T) {
+	cfg, old, fresh := baseConfig(t, "blue_green")
+	handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
+	template := filepath.Join(cfg.Deployment.Root, "handler.template")
+	original := []byte("reverse_proxy 127.0.0.1:8090\n")
+	_ = os.WriteFile(handler, original, 0o644)
+	_ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644)
+	blue := filepath.Join(cfg.Deployment.Root, "slots", "blue")
+	green := filepath.Join(cfg.Deployment.Root, "slots", "green")
+	_ = replaceSymlink(blue, old)
+	_ = replaceSymlink(green, old)
+	cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}}
+	operator := &fakeOperator{active: map[string]bool{}, failPublicAfter: 3}
+	if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err == nil || !strings.Contains(err.Error(), "continuity") {
+		t.Fatalf("expected continuity failure, got %v", err)
+	}
+	body, _ := os.ReadFile(handler)
+	if string(body) != string(original) {
+		t.Fatalf("handler not restored: %q", body)
+	}
+	target, err := resolveReleaseLink(cfg.Deployment.Root, green)
+	if err != nil || target != old {
+		t.Fatalf("green=%q err=%v", target, err)
 	}
 }
 func (f *fakeOperator) ValidateCaddy(context.Context, string) error { return nil }
 func (f *fakeOperator) ReloadCaddy(context.Context) error {
-	if f.failReload {
+	f.reloads++
+	if f.failReload || (f.failReloadAt > 0 && f.reloads == f.failReloadAt) {
 		return errors.New("injected reload failure")
 	}
 	return nil
@@ -117,7 +157,7 @@ func baseConfig(t *testing.T, strategy string) (config.Config, string, string) {
 	if err := os.MkdirAll(filepath.Join(root, "releases"), 0o755); err != nil {
 		t.Fatal(err)
 	}
-	old := filepath.Join(root, "releases", "legacy-old")
+	old := filepath.Join(root, "releases", "sha256-"+strings.Repeat("c", 64))
 	fresh := filepath.Join(root, "releases", "sha256-"+strings.Repeat("a", 64))
 	for _, dir := range []string{old, fresh} {
 		if err := os.Mkdir(dir, 0o755); err != nil {
@@ -127,11 +167,34 @@ func baseConfig(t *testing.T, strategy string) (config.Config, string, string) {
 			t.Fatal(err)
 		}
 	}
-	cfg := config.Config{SchemaVersion: 2, Service: config.Service{Name: "example-site", AllowedHost: "example.test", EnvironmentFile: "/etc/tend/environment/example-site.env"}, Build: config.Build{Package: "./cmd/site", Binary: "app", Branch: "main"}, Deployment: config.Deployment{Strategy: strategy, Root: root, LockFile: filepath.Join(root, "deploy.lock"), StateFile: filepath.Join(root, "state.json"), HealthPath: "/healthz", ReadinessPath: "/readyz", CandidateTimeoutSecs: 2, Smoke: []config.Smoke{{Path: "/", Contains: "Example"}}, PublicSmoke: []config.PublicSmoke{{URL: "https://example.test/", Contains: "Example"}}}}
+	cfg := config.Config{SchemaVersion: 2, Service: config.Service{Name: "example-site", AllowedHost: "example.test", EnvironmentFile: "/etc/tend/environment/example-site.env"}, Build: config.Build{Package: "./cmd/site", Binary: "app", Branch: "main"}, Deployment: config.Deployment{Strategy: strategy, Root: root, LockFile: filepath.Join(root, "deploy.lock"), StateFile: filepath.Join(root, "state.json"), EventLog: filepath.Join(root, "deployment-events.jsonl"), HealthPath: "/healthz", ReadinessPath: "/readyz", CandidateTimeoutSecs: 2, ActivationWindowSecs: 1, Smoke: []config.Smoke{{Path: "/", Contains: "Example"}}, PublicSmoke: []config.PublicSmoke{{URL: "https://example.test/", Contains: "Example"}}}}
 	return cfg, old, fresh
 }
 func manager(operator Operator, fresh string) Manager {
-	return Manager{Operator: operator, Now: func() time.Time { return time.Unix(100, 0).UTC() }, Prepare: func(config.Config, string, string, string) (string, error) { return fresh, nil }, Inspect: func(config.Config, string, string, string) error { return nil }}
+	return Manager{Operator: operator, Now: func() time.Time { return time.Unix(100, 0).UTC() }, Prepare: func(config.Config, string, string, string) (string, error) { return fresh, nil }, Inspect: func(config.Config, string, string, string) error { return nil }, ReadIdentity: func(string) (releaseIdentity, error) {
+		return releaseIdentity{Version: "v0.2.0-preview.1", Commit: strings.Repeat("b", 40)}, nil
+	}, OperationID: func() (string, error) { return strings.Repeat("d", 32), nil }, Sleep: func(context.Context, time.Duration) error { return nil }}
+}
+
+func singletonSettings(t *testing.T, cfg config.Config, currentRelease string) (*config.Singleton, string) {
+	t.Helper()
+	handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
+	template := filepath.Join(cfg.Deployment.Root, "handler.template")
+	if err := os.WriteFile(handler, []byte("reverse_proxy 127.0.0.1:8092\n"), 0o640); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	current := filepath.Join(cfg.Deployment.Root, "current")
+	if err := replaceSymlink(current, currentRelease); err != nil {
+		t.Fatal(err)
+	}
+	return &config.Singleton{
+		Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN",
+		CurrentLink: current, PreviousLink: filepath.Join(cfg.Deployment.Root, "previous"), CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"),
+		CaddyHandler: handler, CaddyHandlerTemplate: template,
+	}, handler
 }

 func TestBlueGreenActivationAndRollback(t *testing.T) {
@@ -158,13 +221,28 @@ func TestBlueGreenActivationAndRollback(t *testing.T) {
 	}
 	operator := &fakeOperator{active: map[string]bool{"example-blue.service": true, "example-green.service": true}}
 	m := manager(operator, fresh)
-	report, err := m.Deploy(context.Background(), cfg, Request{Activate: true})
+	var events []eventlog.Event
+	m.AppendEvent = func(_ string, event eventlog.Event) error {
+		events = append(events, event)
+		return nil
+	}
+	report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
 	if err != nil {
 		t.Fatal(err)
 	}
 	if report.ActiveRelease != fresh || report.PreviousRelease != old {
 		t.Fatalf("report=%+v", report)
 	}
+	deployed, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if deployed.DesiredRelease != fresh || deployed.CandidateRelease != "" || deployed.LastAttemptRelease != fresh || deployed.LastAttemptOutcome != "succeeded" {
+		t.Fatalf("deployment identity state=%+v", deployed)
+	}
+	if len(events) != 2 || events[0].Phase != "candidate" || events[0].Outcome != "running" || events[1].Phase != "activation" || events[1].Outcome != "succeeded" || events[0].OperationID != events[1].OperationID {
+		t.Fatalf("events=%+v", events)
+	}
 	if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 {
 		t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err)
 	}
@@ -180,6 +258,9 @@ func TestBlueGreenActivationAndRollback(t *testing.T) {
 	if record.ActiveRelease != old || record.PreviousRelease != fresh {
 		t.Fatalf("rollback=%+v", record)
 	}
+	if len(events) != 4 || events[2].Phase != "rollback" || events[2].Outcome != "running" || events[3].Phase != "rollback" || events[3].Outcome != "succeeded" || events[2].OperationID != events[3].OperationID || events[2].ArtifactDigest != strings.Repeat("c", 64) {
+		t.Fatalf("rollback events=%+v", events)
+	}
 	if len(operator.probes) != 4 {
 		t.Fatalf("rollback probes=%#v", operator.probes)
 	}
@@ -190,6 +271,58 @@ func TestBlueGreenActivationAndRollback(t *testing.T) {
 	}
 }

+func TestDeploymentEvidenceCanNeverBlockActivationOrRollback(t *testing.T) {
+	cfg, old, fresh := baseConfig(t, "singleton_candidate")
+	cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
+	m := manager(&fakeOperator{active: map[string]bool{}}, fresh)
+	appendCalls := 0
+	m.AppendEvent = func(string, eventlog.Event) error {
+		appendCalls++
+		return errors.New("injected event failure")
+	}
+	report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if report.EventWarnings != 2 || report.ActiveRelease != fresh {
+		t.Fatalf("report=%+v", report)
+	}
+	if _, err := m.Rollback(context.Background(), cfg); err != nil {
+		t.Fatal(err)
+	}
+	record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
+	if err != nil || record.ActiveRelease != old || appendCalls != 4 {
+		t.Fatalf("record=%+v appends=%d err=%v", record, appendCalls, err)
+	}
+}
+
+func TestDeploymentEvidenceIdentityAndEntropyAreBestEffort(t *testing.T) {
+	for _, test := range []struct {
+		name   string
+		damage func(*Manager)
+	}{
+		{"identity", func(manager *Manager) {
+			manager.ReadIdentity = func(string) (releaseIdentity, error) {
+				return releaseIdentity{}, errors.New("injected identity failure")
+			}
+		}},
+		{"entropy", func(manager *Manager) {
+			manager.OperationID = func() (string, error) { return "", errors.New("injected entropy failure") }
+		}},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			cfg, old, fresh := baseConfig(t, "singleton_candidate")
+			cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
+			manager := manager(&fakeOperator{active: map[string]bool{}}, fresh)
+			test.damage(&manager)
+			report, err := manager.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
+			if err != nil || report.EventWarnings != 1 || report.ActiveRelease != fresh {
+				t.Fatalf("report=%+v err=%v", report, err)
+			}
+		})
+	}
+}
+
 func TestBlueGreenCaddyFailureRestoresHandlerAndSlot(t *testing.T) {
 	cfg, old, fresh := baseConfig(t, "blue_green")
 	handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
@@ -215,37 +348,47 @@ func TestBlueGreenCaddyFailureRestoresHandlerAndSlot(t *testing.T) {
 	if err != nil || target != old {
 		t.Fatalf("green=%q err=%v", target, err)
 	}
-	if _, err := os.Stat(cfg.Deployment.StateFile); !os.IsNotExist(err) {
-		t.Fatal("failed activation wrote state")
+	failed, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if failed.ActiveRelease != old || failed.LastAttemptOutcome != "failed" || failed.CandidateRelease != "" {
+		t.Fatalf("failed state=%+v", failed)
 	}
 }

 func TestSingletonRestartFailureRestoresPointers(t *testing.T) {
 	cfg, old, fresh := baseConfig(t, "singleton_candidate")
-	previous := filepath.Join(cfg.Deployment.Root, "previous")
-	current := filepath.Join(cfg.Deployment.Root, "current")
-	_ = replaceSymlink(current, old)
-	cfg.Deployment.Singleton = &config.Singleton{Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN", CurrentLink: current, PreviousLink: previous}
-	operator := &fakeOperator{active: map[string]bool{}, failRestartUnit: "example-site.service"}
+	settings, handler := singletonSettings(t, cfg, old)
+	cfg.Deployment.Singleton = settings
+	operator := &fakeOperator{active: map[string]bool{}, failRestartUnit: "example-site.service", failRestartOnce: true}
 	m := manager(operator, fresh)
 	if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil {
 		t.Fatal("expected failure")
 	}
-	target, err := resolveReleaseLink(cfg.Deployment.Root, current)
+	target, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink)
 	if err != nil || target != old {
 		t.Fatalf("current=%q err=%v", target, err)
 	}
-	if _, err := os.Lstat(previous); !os.IsNotExist(err) {
+	if _, err := os.Lstat(cfg.Deployment.Singleton.PreviousLink); !os.IsNotExist(err) {
 		t.Fatal("previous pointer was not restored")
 	}
+	body, err := os.ReadFile(handler)
+	if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
+		t.Fatalf("handler=%q err=%v", body, err)
+	}
+	if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 {
+		t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err)
+	}
+	if operator.active["example-site-tend-candidate.service"] {
+		t.Fatal("candidate was not stopped after successful restoration")
+	}
 }

-func TestStatePersistsOnlyAfterSuccessfulActivation(t *testing.T) {
+func TestStateRecordsSuccessfulActivation(t *testing.T) {
 	cfg, old, fresh := baseConfig(t, "singleton_candidate")
-	current := filepath.Join(cfg.Deployment.Root, "current")
-	previous := filepath.Join(cfg.Deployment.Root, "previous")
-	_ = replaceSymlink(current, old)
-	cfg.Deployment.Singleton = &config.Singleton{Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN", CurrentLink: current, PreviousLink: previous}
+	settings, handler := singletonSettings(t, cfg, old)
+	cfg.Deployment.Singleton = settings
 	operator := &fakeOperator{active: map[string]bool{}}
 	m := manager(operator, fresh)
 	if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err != nil {
@@ -261,4 +404,72 @@ func TestStatePersistsOnlyAfterSuccessfulActivation(t *testing.T) {
 	if operator.candidateFile != cfg.Service.EnvironmentFile || len(operator.candidateEnvironment) != 1 || operator.candidateEnvironment["EXAMPLE_LISTEN"] != "127.0.0.1:18092" {
 		t.Fatalf("candidate file=%q environment=%#v", operator.candidateFile, operator.candidateEnvironment)
 	}
+	body, err := os.ReadFile(handler)
+	if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
+		t.Fatalf("handler=%q err=%v", body, err)
+	}
+	if operator.reloads != 2 || operator.active["example-site-tend-candidate.service"] {
+		t.Fatalf("reloads=%d active=%#v", operator.reloads, operator.active)
+	}
+}
+
+func TestSingletonContinuityFailureRestoresHandlerPointersAndService(t *testing.T) {
+	cfg, old, fresh := baseConfig(t, "singleton_candidate")
+	settings, handler := singletonSettings(t, cfg, old)
+	cfg.Deployment.Singleton = settings
+	operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failPublicAfter: 4}
+	m := manager(operator, fresh)
+	if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil || !strings.Contains(err.Error(), "continuity") {
+		t.Fatalf("expected continuity failure, got %v", err)
+	}
+	current, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink)
+	if err != nil || current != old {
+		t.Fatalf("current=%q err=%v", current, err)
+	}
+	body, err := os.ReadFile(handler)
+	if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
+		t.Fatalf("handler=%q err=%v", body, err)
+	}
+	if operator.active["example-site-tend-candidate.service"] {
+		t.Fatal("candidate was not stopped after continuity restoration")
+	}
+}
+
+func TestSingletonCaddyReloadFailuresRestorePriorRoute(t *testing.T) {
+	for _, reload := range []int{1, 2} {
+		t.Run(fmt.Sprintf("reload-%d", reload), func(t *testing.T) {
+			cfg, old, fresh := baseConfig(t, "singleton_candidate")
+			settings, handler := singletonSettings(t, cfg, old)
+			cfg.Deployment.Singleton = settings
+			operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failReloadAt: reload}
+			if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true}); err == nil || !strings.Contains(err.Error(), "Caddy reload failed") {
+				t.Fatalf("expected Caddy reload failure, got %v", err)
+			}
+			current, err := resolveReleaseLink(cfg.Deployment.Root, settings.CurrentLink)
+			if err != nil || current != old {
+				t.Fatalf("current=%q err=%v", current, err)
+			}
+			body, err := os.ReadFile(handler)
+			if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
+				t.Fatalf("handler=%q err=%v", body, err)
+			}
+			if operator.active["example-site-tend-candidate.service"] {
+				t.Fatal("candidate was not stopped after route restoration")
+			}
+		})
+	}
+}
+
+func TestSingletonIncompleteRecoveryKeepsProvenCandidateRunning(t *testing.T) {
+	cfg, old, fresh := baseConfig(t, "singleton_candidate")
+	settings, _ := singletonSettings(t, cfg, old)
+	cfg.Deployment.Singleton = settings
+	operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failReload: true}
+	_, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true})
+	if err == nil || !strings.Contains(err.Error(), "candidate remains routed for operator recovery") {
+		t.Fatalf("expected explicit incomplete recovery, got %v", err)
+	}
+	if !operator.active["example-site-tend-candidate.service"] {
+		t.Fatal("proven candidate was stopped despite incomplete route restoration")
+	}
 }