tend/internal/deploy/reconcile.go

// SPDX-License-Identifier: AGPL-3.0-only

package deploy

import (
	"bytes"
	"context"
	"errors"
	"os"

	"gamertan.com/tend/internal/config"
	"gamertan.com/tend/internal/state"
)

type Finding struct {
	Code     string `json:"code"`
	Severity string `json:"severity"`
	Message  string `json:"message"`
}

type ObservedReleaseIdentity struct {
	Version string `json:"version"`
	Commit  string `json:"commit"`
}

type ObservedState struct {
	ActiveRelease         string                   `json:"active_release,omitempty"`
	PreviousRelease       string                   `json:"previous_release,omitempty"`
	ActiveIdentity        *ObservedReleaseIdentity `json:"active_identity,omitempty"`
	Units                 map[string]bool          `json:"units"`
	CandidateLease        bool                     `json:"candidate_lease"`
	CandidateUnit         string                   `json:"candidate_unit,omitempty"`
	CandidateUnitActive   *bool                    `json:"candidate_unit_active,omitempty"`
	LegacyCandidateUnit   string                   `json:"legacy_candidate_unit,omitempty"`
	LegacyCandidateActive *bool                    `json:"legacy_candidate_unit_active,omitempty"`
	RouteHandlerMatches   bool                     `json:"route_handler_matches"`
	HandlerFileTarget     string                   `json:"handler_file_target,omitempty"`
}

type Reconciliation struct {
	Service          string        `json:"service"`
	Strategy         string        `json:"strategy"`
	Mutation         string        `json:"mutation"`
	Consistent       bool          `json:"consistent"`
	StateInitialized bool          `json:"state_initialized"`
	State            *state.Record `json:"state,omitempty"`
	Observed         ObservedState `json:"observed"`
	Disposition      string        `json:"disposition"`
	Findings         []Finding     `json:"findings"`
}

// Reconcile observes configured units, release pointers, installed release
// identity, and the imported Caddy handler. It never acquires the deployment
// lock or mutates service state; proposed repairs remain an operator decision.
func (m Manager) Reconcile(ctx context.Context, cfg config.Config) (Reconciliation, error) {
	if err := cfg.Validate(); err != nil {
		return Reconciliation{}, err
	}
	if m.Operator == nil || m.ReadIdentity == nil {
		return Reconciliation{}, errors.New("reconciliation dependencies are unavailable")
	}
	report := Reconciliation{
		Service: cfg.Service.Name, Strategy: cfg.Deployment.Strategy, Mutation: "none",
		Observed: ObservedState{Units: map[string]bool{}}, Findings: []Finding{},
	}
	add := func(code, severity, message string) {
		report.Findings = append(report.Findings, Finding{Code: code, Severity: severity, Message: message})
	}
	record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
	if err == nil {
		report.State = &record
		report.StateInitialized = true
	} else if os.IsNotExist(err) {
		add("state_uninitialized", "warning", "Tend has no validated state record for this service.")
	} else {
		add("state_invalid", "error", "The Tend state record could not be validated.")
	}

	activeSlot := ""
	activeUnit := ""
	activeAddress := ""
	handler := ""
	template := ""
	var candidateLease *state.CandidateLease
	switch cfg.Deployment.Strategy {
	case "singleton_candidate":
		single := *cfg.Deployment.Singleton
		activeSlot = "singleton"
		activeUnit = single.Unit
		activeAddress = single.Address
		handler, template = single.CaddyHandler, single.CaddyHandlerTemplate
		report.Observed.ActiveRelease = observeReleaseLink(cfg, single.CurrentLink, true, add)
		report.Observed.PreviousRelease = observeReleaseLink(cfg, single.PreviousLink, false, add)
		legacyCandidateUnit := cfg.Service.Name + "-tend-candidate.service"
		candidateUnit := legacyCandidateUnit
		lease, leaseErr := state.LoadCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, cfg.Service.Name)
		if leaseErr == nil {
			candidateLease = &lease
			report.Observed.CandidateLease = true
			candidateUnit = lease.Unit
		} else if !os.IsNotExist(leaseErr) {
			report.Observed.CandidateLease = true
			candidateUnit = ""
			add("candidate_lease_invalid", "error", "The operation-scoped candidate lease could not be validated.")
		} else if report.State != nil && report.State.CandidateRelease != "" {
			report.Observed.CandidateLease = true
			candidateUnit = ""
			add("legacy_candidate_lease", "error", "The state records a candidate release without an operation-scoped lease and requires manual review.")
		}
		if candidateUnit != "" {
			report.Observed.CandidateUnit = candidateUnit
			candidateActive, candidateErr := m.Operator.IsActive(ctx, candidateUnit)
			if candidateErr != nil {
				add("candidate_unit_unobservable", "error", "The transient candidate unit state could not be observed.")
			} else {
				report.Observed.CandidateUnitActive = &candidateActive
				report.Observed.Units[candidateUnit] = candidateActive
			}
		}
		if candidateUnit != legacyCandidateUnit {
			report.Observed.LegacyCandidateUnit = legacyCandidateUnit
			legacyActive, legacyErr := m.Operator.IsActive(ctx, legacyCandidateUnit)
			if legacyErr != nil {
				add("legacy_candidate_unit_unobservable", "error", "The legacy fixed candidate unit state could not be observed.")
			} else {
				report.Observed.LegacyCandidateActive = &legacyActive
				report.Observed.Units[legacyCandidateUnit] = legacyActive
				if legacyActive {
					add("legacy_candidate_unit_active", "error", "A legacy fixed-name candidate remains active beside an operation-scoped lease.")
				}
			}
		}
	case "blue_green":
		blueGreen := *cfg.Deployment.BlueGreen
		handler, template = blueGreen.CaddyHandler, blueGreen.CaddyHandlerTemplate
		activeSlot = blueGreen.BootstrapActive
		if report.State != nil {
			activeSlot = report.State.ActiveSlot
		}
		active := slotConfig(blueGreen, activeSlot)
		previousName := "blue"
		if activeSlot == "blue" {
			previousName = "green"
		}
		previous := slotConfig(blueGreen, previousName)
		activeUnit, activeAddress = active.Unit, active.Address
		report.Observed.ActiveRelease = observeReleaseLink(cfg, active.Link, true, add)
		report.Observed.PreviousRelease = observeReleaseLink(cfg, previous.Link, false, add)
		for _, slot := range []config.Slot{blueGreen.Blue, blueGreen.Green} {
			observeUnit(ctx, m.Operator, slot.Unit, report.Observed.Units, add)
		}
	}
	if _, exists := report.Observed.Units[activeUnit]; !exists {
		observeUnit(ctx, m.Operator, activeUnit, report.Observed.Units, add)
	}
	if active, observed := report.Observed.Units[activeUnit]; activeUnit != "" && observed && !active {
		add("active_unit_inactive", "error", "The configured active service unit is not active.")
	}

	if report.Observed.ActiveRelease != "" {
		identity, identityErr := m.ReadIdentity(report.Observed.ActiveRelease)
		if identityErr != nil {
			add("active_identity_unreadable", "error", "The observed active release identity could not be validated.")
		} else {
			report.Observed.ActiveIdentity = &ObservedReleaseIdentity{Version: identity.Version, Commit: identity.Commit}
		}
	}
	expectedHandler, renderErr := renderHandler(template, activeAddress)
	actualHandler, readErr := os.ReadFile(handler)
	if renderErr != nil || readErr != nil {
		add("route_handler_unreadable", "error", "The configured Caddy handler or its template could not be validated.")
	} else {
		report.Observed.RouteHandlerMatches = bytes.Equal(expectedHandler, actualHandler)
		if report.Observed.RouteHandlerMatches {
			report.Observed.HandlerFileTarget = "installed"
		} else if cfg.Deployment.Strategy == "singleton_candidate" {
			candidateAddress := cfg.Deployment.Singleton.CandidateAddress
			if candidateLease != nil {
				candidateAddress = candidateLease.Address
			}
			candidateHandler, candidateErr := renderHandler(template, candidateAddress)
			if candidateErr == nil && bytes.Equal(candidateHandler, actualHandler) {
				report.Observed.HandlerFileTarget = "candidate"
			} else {
				report.Observed.HandlerFileTarget = "other"
			}
		}
		if !report.Observed.RouteHandlerMatches {
			add("route_handler_drift", "error", "The installed Caddy handler does not match the configured active upstream.")
		}
	}

	if report.State != nil {
		if report.State.ActiveSlot != activeSlot || report.State.ActiveRelease != report.Observed.ActiveRelease {
			add("active_release_drift", "error", "Recorded active state does not match the observed active release pointer.")
		}
		if report.State.PreviousRelease != report.Observed.PreviousRelease {
			add("previous_release_drift", "warning", "Recorded rollback state does not match the observed previous release pointer.")
		}
		if cfg.Deployment.Strategy == "singleton_candidate" {
			leased := report.Observed.CandidateLease && candidateLease != nil
			candidateActive := report.Observed.CandidateUnitActive != nil && *report.Observed.CandidateUnitActive
			if candidateLease != nil && report.State.CandidateRelease == "" {
				add("candidate_lease_without_running_attempt", "error", "An operation-scoped candidate lease remains after the recorded attempt settled.")
			}
			if candidateLease != nil && report.State.CandidateRelease != "" && candidateLease.Release != report.State.CandidateRelease {
				add("candidate_lease_release_mismatch", "error", "The operation-scoped candidate lease does not match the recorded candidate release.")
			}
			if leased && report.Observed.CandidateUnitActive != nil && !candidateActive {
				add("inactive_candidate_lease", "error", "State retains a candidate lease but its operation-scoped unit is inactive.")
			}
			if !report.Observed.CandidateLease && candidateActive {
				add("unleased_candidate_active", "error", "A transient candidate unit is active without a matching running attempt.")
			}
			if leased && candidateActive && report.Observed.HandlerFileTarget == "candidate" {
				add("retained_candidate_routed", "warning", "The retained candidate appears in the handler file; do not stop it before establishing another healthy route.")
			}
			if leased && candidateActive && report.Observed.HandlerFileTarget != "candidate" {
				add("candidate_active_not_routed", "warning", "The leased candidate is active but the handler file does not target it; cleanup remains an explicit reviewed operation.")
			}
		}
	}
	switch {
	case !report.StateInitialized:
		report.Disposition = "state_uninitialized"
	case report.Observed.CandidateLease && report.Observed.CandidateUnit == "":
		report.Disposition = "legacy_or_invalid_candidate_lease"
	case report.Observed.CandidateLease && report.Observed.CandidateUnitActive != nil && *report.Observed.CandidateUnitActive && report.Observed.HandlerFileTarget == "candidate":
		report.Disposition = "retained_candidate_handler_file"
	case report.Observed.CandidateLease && report.Observed.CandidateUnitActive != nil && *report.Observed.CandidateUnitActive:
		report.Disposition = "candidate_active_not_in_handler_file"
	case report.Observed.CandidateLease:
		report.Disposition = "inactive_candidate_lease"
	case len(report.Findings) == 0:
		report.Disposition = "settled"
	default:
		report.Disposition = "manual_review_required"
	}
	report.Consistent = len(report.Findings) == 0
	return report, nil
}

func observeReleaseLink(cfg config.Config, link string, required bool, add func(string, string, string)) string {
	release, err := resolveReleaseLink(cfg.Deployment.Root, link)
	if err == nil {
		return release
	}
	if !required && os.IsNotExist(err) {
		return ""
	}
	code := "previous_pointer_unreadable"
	message := "The configured previous release pointer could not be validated."
	severity := "warning"
	if required {
		code = "active_pointer_unreadable"
		message = "The configured active release pointer could not be validated."
		severity = "error"
	}
	add(code, severity, message)
	return ""
}

func observeUnit(ctx context.Context, operator Operator, unit string, units map[string]bool, add func(string, string, string)) {
	active, err := operator.IsActive(ctx, unit)
	if err != nil {
		add("unit_unobservable", "error", "A configured service unit state could not be observed.")
		return
	}
	units[unit] = active
}