docs: publish Tend Compose continuity evidence

Export the reviewed allowlisted snapshot from private source commit 07c1655921f21ee5e4fc4d85639d199e8867b17d. This records the Docker Compose activation, schema-compatible rollback, and stateful migration resource findings from Observatory Preview 19 dogfooding. AI-Assisted: OpenAI Codex Signed-off-by: Cole Speelman <crspeelman@gmail.com>
2026-08-18 21:42:33 -04:00
commit bf56dbce0f
83 changed files with 8555 additions and 0 deletions
@@ -0,0 +1,270 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+
+package deploy
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"os"
+
+	"gamertan.com/tend/internal/config"
+	"gamertan.com/tend/internal/state"
+)
+
+type Finding struct {
+	Code     string `json:"code"`
+	Severity string `json:"severity"`
+	Message  string `json:"message"`
+}
+
+type ObservedReleaseIdentity struct {
+	Version string `json:"version"`
+	Commit  string `json:"commit"`
+}
+
+type ObservedState struct {
+	ActiveRelease         string                   `json:"active_release,omitempty"`
+	PreviousRelease       string                   `json:"previous_release,omitempty"`
+	ActiveIdentity        *ObservedReleaseIdentity `json:"active_identity,omitempty"`
+	Units                 map[string]bool          `json:"units"`
+	CandidateLease        bool                     `json:"candidate_lease"`
+	CandidateUnit         string                   `json:"candidate_unit,omitempty"`
+	CandidateUnitActive   *bool                    `json:"candidate_unit_active,omitempty"`
+	LegacyCandidateUnit   string                   `json:"legacy_candidate_unit,omitempty"`
+	LegacyCandidateActive *bool                    `json:"legacy_candidate_unit_active,omitempty"`
+	RouteHandlerMatches   bool                     `json:"route_handler_matches"`
+	HandlerFileTarget     string                   `json:"handler_file_target,omitempty"`
+}
+
+type Reconciliation struct {
+	Service          string        `json:"service"`
+	Strategy         string        `json:"strategy"`
+	Mutation         string        `json:"mutation"`
+	Consistent       bool          `json:"consistent"`
+	StateInitialized bool          `json:"state_initialized"`
+	State            *state.Record `json:"state,omitempty"`
+	Observed         ObservedState `json:"observed"`
+	Disposition      string        `json:"disposition"`
+	Findings         []Finding     `json:"findings"`
+}
+
+// Reconcile observes configured units, release pointers, installed release
+// identity, and the imported Caddy handler. It never acquires the deployment
+// lock or mutates service state; proposed repairs remain an operator decision.
+func (m Manager) Reconcile(ctx context.Context, cfg config.Config) (Reconciliation, error) {
+	if err := cfg.Validate(); err != nil {
+		return Reconciliation{}, err
+	}
+	if m.Operator == nil || m.ReadIdentity == nil {
+		return Reconciliation{}, errors.New("reconciliation dependencies are unavailable")
+	}
+	report := Reconciliation{
+		Service: cfg.Service.Name, Strategy: cfg.Deployment.Strategy, Mutation: "none",
+		Observed: ObservedState{Units: map[string]bool{}}, Findings: []Finding{},
+	}
+	add := func(code, severity, message string) {
+		report.Findings = append(report.Findings, Finding{Code: code, Severity: severity, Message: message})
+	}
+	record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
+	if err == nil {
+		report.State = &record
+		report.StateInitialized = true
+	} else if os.IsNotExist(err) {
+		add("state_uninitialized", "warning", "Tend has no validated state record for this service.")
+	} else {
+		add("state_invalid", "error", "The Tend state record could not be validated.")
+	}
+
+	activeSlot := ""
+	activeUnit := ""
+	activeAddress := ""
+	handler := ""
+	template := ""
+	var candidateLease *state.CandidateLease
+	switch cfg.Deployment.Strategy {
+	case "singleton_candidate":
+		single := *cfg.Deployment.Singleton
+		activeSlot = "singleton"
+		activeUnit = single.Unit
+		activeAddress = single.Address
+		handler, template = single.CaddyHandler, single.CaddyHandlerTemplate
+		report.Observed.ActiveRelease = observeReleaseLink(cfg, single.CurrentLink, true, add)
+		report.Observed.PreviousRelease = observeReleaseLink(cfg, single.PreviousLink, false, add)
+		legacyCandidateUnit := cfg.Service.Name + "-tend-candidate.service"
+		candidateUnit := legacyCandidateUnit
+		lease, leaseErr := state.LoadCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, cfg.Service.Name)
+		if leaseErr == nil {
+			candidateLease = &lease
+			report.Observed.CandidateLease = true
+			candidateUnit = lease.Unit
+		} else if !os.IsNotExist(leaseErr) {
+			report.Observed.CandidateLease = true
+			candidateUnit = ""
+			add("candidate_lease_invalid", "error", "The operation-scoped candidate lease could not be validated.")
+		} else if report.State != nil && report.State.CandidateRelease != "" {
+			report.Observed.CandidateLease = true
+			candidateUnit = ""
+			add("legacy_candidate_lease", "error", "The state records a candidate release without an operation-scoped lease and requires manual review.")
+		}
+		if candidateUnit != "" {
+			report.Observed.CandidateUnit = candidateUnit
+			candidateActive, candidateErr := m.Operator.IsActive(ctx, candidateUnit)
+			if candidateErr != nil {
+				add("candidate_unit_unobservable", "error", "The transient candidate unit state could not be observed.")
+			} else {
+				report.Observed.CandidateUnitActive = &candidateActive
+				report.Observed.Units[candidateUnit] = candidateActive
+			}
+		}
+		if candidateUnit != legacyCandidateUnit {
+			report.Observed.LegacyCandidateUnit = legacyCandidateUnit
+			legacyActive, legacyErr := m.Operator.IsActive(ctx, legacyCandidateUnit)
+			if legacyErr != nil {
+				add("legacy_candidate_unit_unobservable", "error", "The legacy fixed candidate unit state could not be observed.")
+			} else {
+				report.Observed.LegacyCandidateActive = &legacyActive
+				report.Observed.Units[legacyCandidateUnit] = legacyActive
+				if legacyActive {
+					add("legacy_candidate_unit_active", "error", "A legacy fixed-name candidate remains active beside an operation-scoped lease.")
+				}
+			}
+		}
+	case "blue_green":
+		blueGreen := *cfg.Deployment.BlueGreen
+		handler, template = blueGreen.CaddyHandler, blueGreen.CaddyHandlerTemplate
+		activeSlot = blueGreen.BootstrapActive
+		if report.State != nil {
+			activeSlot = report.State.ActiveSlot
+		}
+		active := slotConfig(blueGreen, activeSlot)
+		previousName := "blue"
+		if activeSlot == "blue" {
+			previousName = "green"
+		}
+		previous := slotConfig(blueGreen, previousName)
+		activeUnit, activeAddress = active.Unit, active.Address
+		report.Observed.ActiveRelease = observeReleaseLink(cfg, active.Link, true, add)
+		report.Observed.PreviousRelease = observeReleaseLink(cfg, previous.Link, false, add)
+		for _, slot := range []config.Slot{blueGreen.Blue, blueGreen.Green} {
+			observeUnit(ctx, m.Operator, slot.Unit, report.Observed.Units, add)
+		}
+	}
+	if _, exists := report.Observed.Units[activeUnit]; !exists {
+		observeUnit(ctx, m.Operator, activeUnit, report.Observed.Units, add)
+	}
+	if active, observed := report.Observed.Units[activeUnit]; activeUnit != "" && observed && !active {
+		add("active_unit_inactive", "error", "The configured active service unit is not active.")
+	}
+
+	if report.Observed.ActiveRelease != "" {
+		identity, identityErr := m.ReadIdentity(report.Observed.ActiveRelease)
+		if identityErr != nil {
+			add("active_identity_unreadable", "error", "The observed active release identity could not be validated.")
+		} else {
+			report.Observed.ActiveIdentity = &ObservedReleaseIdentity{Version: identity.Version, Commit: identity.Commit}
+		}
+	}
+	expectedHandler, renderErr := renderHandler(template, activeAddress)
+	actualHandler, readErr := os.ReadFile(handler)
+	if renderErr != nil || readErr != nil {
+		add("route_handler_unreadable", "error", "The configured Caddy handler or its template could not be validated.")
+	} else {
+		report.Observed.RouteHandlerMatches = bytes.Equal(expectedHandler, actualHandler)
+		if report.Observed.RouteHandlerMatches {
+			report.Observed.HandlerFileTarget = "installed"
+		} else if cfg.Deployment.Strategy == "singleton_candidate" {
+			candidateAddress := cfg.Deployment.Singleton.CandidateAddress
+			if candidateLease != nil {
+				candidateAddress = candidateLease.Address
+			}
+			candidateHandler, candidateErr := renderHandler(template, candidateAddress)
+			if candidateErr == nil && bytes.Equal(candidateHandler, actualHandler) {
+				report.Observed.HandlerFileTarget = "candidate"
+			} else {
+				report.Observed.HandlerFileTarget = "other"
+			}
+		}
+		if !report.Observed.RouteHandlerMatches {
+			add("route_handler_drift", "error", "The installed Caddy handler does not match the configured active upstream.")
+		}
+	}
+
+	if report.State != nil {
+		if report.State.ActiveSlot != activeSlot || report.State.ActiveRelease != report.Observed.ActiveRelease {
+			add("active_release_drift", "error", "Recorded active state does not match the observed active release pointer.")
+		}
+		if report.State.PreviousRelease != report.Observed.PreviousRelease {
+			add("previous_release_drift", "warning", "Recorded rollback state does not match the observed previous release pointer.")
+		}
+		if cfg.Deployment.Strategy == "singleton_candidate" {
+			leased := report.Observed.CandidateLease && candidateLease != nil
+			candidateActive := report.Observed.CandidateUnitActive != nil && *report.Observed.CandidateUnitActive
+			if candidateLease != nil && report.State.CandidateRelease == "" {
+				add("candidate_lease_without_running_attempt", "error", "An operation-scoped candidate lease remains after the recorded attempt settled.")
+			}
+			if candidateLease != nil && report.State.CandidateRelease != "" && candidateLease.Release != report.State.CandidateRelease {
+				add("candidate_lease_release_mismatch", "error", "The operation-scoped candidate lease does not match the recorded candidate release.")
+			}
+			if leased && report.Observed.CandidateUnitActive != nil && !candidateActive {
+				add("inactive_candidate_lease", "error", "State retains a candidate lease but its operation-scoped unit is inactive.")
+			}
+			if !report.Observed.CandidateLease && candidateActive {
+				add("unleased_candidate_active", "error", "A transient candidate unit is active without a matching running attempt.")
+			}
+			if leased && candidateActive && report.Observed.HandlerFileTarget == "candidate" {
+				add("retained_candidate_routed", "warning", "The retained candidate appears in the handler file; do not stop it before establishing another healthy route.")
+			}
+			if leased && candidateActive && report.Observed.HandlerFileTarget != "candidate" {
+				add("candidate_active_not_routed", "warning", "The leased candidate is active but the handler file does not target it; cleanup remains an explicit reviewed operation.")
+			}
+		}
+	}
+	switch {
+	case !report.StateInitialized:
+		report.Disposition = "state_uninitialized"
+	case report.Observed.CandidateLease && report.Observed.CandidateUnit == "":
+		report.Disposition = "legacy_or_invalid_candidate_lease"
+	case report.Observed.CandidateLease && report.Observed.CandidateUnitActive != nil && *report.Observed.CandidateUnitActive && report.Observed.HandlerFileTarget == "candidate":
+		report.Disposition = "retained_candidate_handler_file"
+	case report.Observed.CandidateLease && report.Observed.CandidateUnitActive != nil && *report.Observed.CandidateUnitActive:
+		report.Disposition = "candidate_active_not_in_handler_file"
+	case report.Observed.CandidateLease:
+		report.Disposition = "inactive_candidate_lease"
+	case len(report.Findings) == 0:
+		report.Disposition = "settled"
+	default:
+		report.Disposition = "manual_review_required"
+	}
+	report.Consistent = len(report.Findings) == 0
+	return report, nil
+}
+
+func observeReleaseLink(cfg config.Config, link string, required bool, add func(string, string, string)) string {
+	release, err := resolveReleaseLink(cfg.Deployment.Root, link)
+	if err == nil {
+		return release
+	}
+	if !required && os.IsNotExist(err) {
+		return ""
+	}
+	code := "previous_pointer_unreadable"
+	message := "The configured previous release pointer could not be validated."
+	severity := "warning"
+	if required {
+		code = "active_pointer_unreadable"
+		message = "The configured active release pointer could not be validated."
+		severity = "error"
+	}
+	add(code, severity, message)
+	return ""
+}
+
+func observeUnit(ctx context.Context, operator Operator, unit string, units map[string]bool, add func(string, string, string)) {
+	active, err := operator.IsActive(ctx, unit)
+	if err != nil {
+		add("unit_unobservable", "error", "A configured service unit state could not be observed.")
+		return
+	}
+	units[unit] = active
+}