feat: publish Tend v0.2 Preview 2 source

Export the reviewed allowlisted snapshot from private source commit 8aab3db43f35e6a49aa497f45d73701b13fc9f32 and tree 992132ea4703437dc13ffdbb04a077816c02caf9. This includes routed singleton continuity, deployment evidence, strict schema-2 configuration, restricted transport, and the independently compilable public-tree guard.

AI-Assisted: OpenAI Codex
Signed-off-by: Cole Speelman <crspeelman@gmail.com>
This commit is contained in:
2026-08-18 06:40:58 -04:00
parent 00d1dd4209
commit 9d9fc83dd0
33 changed files with 1463 additions and 150 deletions
+292 -76
View File
@@ -14,6 +14,7 @@ import (
"time"
"gamertan.com/tend/internal/config"
"gamertan.com/tend/internal/eventlog"
"gamertan.com/tend/internal/state"
)
@@ -29,6 +30,7 @@ type Report struct {
Release string `json:"release,omitempty"`
ActiveRelease string `json:"active_release,omitempty"`
PreviousRelease string `json:"previous_release,omitempty"`
EventWarnings int `json:"event_warnings,omitempty"`
}
type Status struct {
State *state.Record `json:"state,omitempty"`
@@ -37,14 +39,18 @@ type Status struct {
}
type Manager struct {
Operator Operator
Now func() time.Time
Prepare func(config.Config, string, string, string) (string, error)
Inspect func(config.Config, string, string, string) error
Operator Operator
Now func() time.Time
Prepare func(config.Config, string, string, string) (string, error)
Inspect func(config.Config, string, string, string) error
ReadIdentity func(string) (releaseIdentity, error)
OperationID func() (string, error)
AppendEvent func(string, eventlog.Event) error
Sleep func(context.Context, time.Duration) error
}
func NewManager(operator Operator) Manager {
return Manager{Operator: operator, Now: time.Now, Prepare: prepareRelease, Inspect: inspectArtifact}
return Manager{Operator: operator, Now: time.Now, Prepare: prepareRelease, Inspect: inspectArtifact, ReadIdentity: readReleaseIdentity, OperationID: eventlog.OperationID, AppendEvent: eventlog.Append, Sleep: sleepContext}
}
func (m Manager) Deploy(ctx context.Context, cfg config.Config, request Request) (Report, error) {
@@ -66,10 +72,44 @@ func (m Manager) Deploy(ctx context.Context, cfg config.Config, request Request)
if err != nil {
return Report{}, err
}
started := m.Now()
eventWarnings := 0
identity, identityErr := m.ReadIdentity(release)
if identityErr != nil {
eventWarnings++
}
operationID := ""
if m.OperationID != nil {
operationID, err = m.OperationID()
if err != nil {
eventWarnings++
operationID = ""
}
}
emit := func(phase, slot, outcome string) {
if m.AppendEvent == nil || identityErr != nil || operationID == "" {
return
}
event := eventlog.Event{Version: eventlog.Version, OperationID: operationID, Service: cfg.Service.Name, ArtifactDigest: request.ApprovedSHA256, Commit: identity.Commit, ReleaseVersion: identity.Version, Phase: phase, Slot: slot, DurationMillis: max(0, m.Now().Sub(started).Milliseconds()), Outcome: outcome, ObservedAt: m.Now().UTC().Format(time.RFC3339Nano)}
if eventErr := m.AppendEvent(cfg.Deployment.EventLog, event); eventErr != nil {
eventWarnings++
}
}
record, err := loadOrBootstrap(cfg, m.Now())
if err != nil {
return Report{}, err
}
attemptAt := m.Now().UTC().Format(time.RFC3339)
record.DesiredRelease = release
record.CandidateRelease = release
record.LastAttemptRelease = release
record.LastAttemptOutcome = "running"
record.LastAttemptAt = attemptAt
record.UpdatedAt = attemptAt
if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil {
return Report{}, err
}
emit("candidate", inactiveSlot(cfg, record), "running")
switch cfg.Deployment.Strategy {
case "blue_green":
err = m.deployBlueGreen(ctx, cfg, record, release)
@@ -79,13 +119,64 @@ func (m Manager) Deploy(ctx context.Context, cfg config.Config, request Request)
err = errors.New("unsupported strategy")
}
if err != nil {
return Report{}, err
failed := record
failed.CandidateRelease = ""
failed.LastAttemptOutcome = "failed"
failed.UpdatedAt = m.Now().UTC().Format(time.RFC3339)
_ = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, failed)
emit("activation", inactiveSlot(cfg, record), "failed")
return Report{EventWarnings: eventWarnings}, err
}
emit("activation", inactiveSlot(cfg, record), "succeeded")
updated, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
if err != nil {
return Report{}, err
}
return Report{Validated: true, Mutation: "activated", Release: release, ActiveRelease: updated.ActiveRelease, PreviousRelease: updated.PreviousRelease}, nil
return Report{Validated: true, Mutation: "activated", Release: release, ActiveRelease: updated.ActiveRelease, PreviousRelease: updated.PreviousRelease, EventWarnings: eventWarnings}, nil
}
type releaseIdentity struct {
Version string `json:"version"`
Commit string `json:"commit"`
}
func readReleaseIdentity(release string) (releaseIdentity, error) {
b, err := os.ReadFile(filepath.Join(release, "RELEASE.json"))
if err != nil {
return releaseIdentity{}, fmt.Errorf("read installed release identity: %w", err)
}
if len(b) > 1<<20 {
return releaseIdentity{}, errors.New("installed release identity is too large")
}
var identity releaseIdentity
if err := json.Unmarshal(b, &identity); err != nil {
return releaseIdentity{}, errors.New("decode installed release identity")
}
if identity.Version == "" || identity.Commit == "" {
return releaseIdentity{}, errors.New("installed release identity is incomplete")
}
return identity, nil
}
func inactiveSlot(cfg config.Config, record state.Record) string {
if cfg.Deployment.Strategy == "singleton_candidate" {
return "singleton"
}
if record.ActiveSlot == "blue" {
return "green"
}
return "blue"
}
func sleepContext(ctx context.Context, duration time.Duration) error {
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}
func loadOrBootstrap(cfg config.Config, now time.Time) (state.Record, error) {
@@ -103,13 +194,13 @@ func loadOrBootstrap(cfg config.Config, now time.Time) (state.Record, error) {
if err != nil {
return state.Record{}, fmt.Errorf("bootstrap active slot: %w", err)
}
return state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, ActiveSlot: slot, ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil
return state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: slot, ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil
case "singleton_candidate":
release, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink)
if err != nil {
return state.Record{}, fmt.Errorf("bootstrap singleton: %w", err)
}
return state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, ActiveSlot: "singleton", ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil
return state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil
}
return state.Record{}, errors.New("unsupported strategy")
}
@@ -180,7 +271,11 @@ func (m Manager) deployBlueGreen(ctx context.Context, cfg config.Config, record
if err = m.probePublic(ctx, cfg, true); err != nil {
return fmt.Errorf("public-origin smoke failed: %w", err)
}
next := state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, ActiveSlot: inactive, ActiveRelease: release, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
previous := slotConfig(bg, record.ActiveSlot)
if err = m.continuityWindow(ctx, cfg, previous.Address, true); err != nil {
return fmt.Errorf("activation continuity failed: %w", err)
}
next := state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: inactive, ActiveRelease: release, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, LastAttemptRelease: release, LastAttemptOutcome: "succeeded", LastAttemptAt: record.LastAttemptAt, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
if err = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next); err != nil {
return err
}
@@ -188,62 +283,10 @@ func (m Manager) deployBlueGreen(ctx context.Context, cfg config.Config, record
}
func (m Manager) deploySingleton(ctx context.Context, cfg config.Config, record state.Record, release string) (err error) {
single := *cfg.Deployment.Singleton
candidateUnit := cfg.Service.Name + "-tend-candidate.service"
env := map[string]string{single.ListenEnv: single.CandidateAddress}
binary := filepath.Join(release, cfg.Build.Binary)
if err = m.Operator.StartCandidate(ctx, candidateUnit, binary, cfg.Service.EnvironmentFile, env); err != nil {
if err = m.activateSingletonRelease(ctx, cfg, release, true); err != nil {
return err
}
defer func() {
stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = m.Operator.Stop(stopCtx, candidateUnit)
}()
if err = m.probeAll(ctx, cfg, single.CandidateAddress); err != nil {
return fmt.Errorf("candidate failed: %w", err)
}
oldCurrent, err := resolveReleaseLink(cfg.Deployment.Root, single.CurrentLink)
if err != nil {
return err
}
oldPrevious, previousErr := resolveReleaseLink(cfg.Deployment.Root, single.PreviousLink)
currentChanged := false
previousChanged := false
defer func() {
if err == nil {
return
}
if currentChanged {
_ = replaceSymlink(single.CurrentLink, oldCurrent)
_ = m.Operator.Restart(ctx, single.Unit)
}
if previousChanged {
if previousErr == nil {
_ = replaceSymlink(single.PreviousLink, oldPrevious)
} else {
_ = removeSymlink(single.PreviousLink)
}
}
}()
if err = replaceSymlink(single.PreviousLink, oldCurrent); err != nil {
return err
}
previousChanged = true
if err = replaceSymlink(single.CurrentLink, release); err != nil {
return err
}
currentChanged = true
if err = m.Operator.Restart(ctx, single.Unit); err != nil {
return err
}
if err = m.probeAll(ctx, cfg, single.Address); err != nil {
return fmt.Errorf("post-activation smoke failed: %w", err)
}
if err = m.probePublic(ctx, cfg, true); err != nil {
return fmt.Errorf("public-origin smoke failed: %w", err)
}
next := state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, ActiveSlot: "singleton", ActiveRelease: release, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
next := state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, LastAttemptRelease: release, LastAttemptOutcome: "succeeded", LastAttemptAt: record.LastAttemptAt, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
if err = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next); err != nil {
return err
}
@@ -263,6 +306,21 @@ func (m Manager) Rollback(ctx context.Context, cfg config.Config) (state.Record,
if record.PreviousRelease == "" {
return state.Record{}, errors.New("no previous release is recorded")
}
started := m.Now()
identity, identityErr := m.ReadIdentity(record.PreviousRelease)
digest, digestErr := releaseDigest(record.PreviousRelease)
operationID := ""
if m.OperationID != nil {
operationID, _ = m.OperationID()
}
emit := func(outcome string) {
if m.AppendEvent == nil || identityErr != nil || digestErr != nil || operationID == "" {
return
}
event := eventlog.Event{Version: eventlog.Version, OperationID: operationID, Service: cfg.Service.Name, ArtifactDigest: digest, Commit: identity.Commit, ReleaseVersion: identity.Version, Phase: "rollback", Slot: record.PreviousSlot, DurationMillis: max(0, m.Now().Sub(started).Milliseconds()), Outcome: outcome, ObservedAt: m.Now().UTC().Format(time.RFC3339Nano)}
_ = m.AppendEvent(cfg.Deployment.EventLog, event)
}
emit("running")
switch cfg.Deployment.Strategy {
case "blue_green":
err = m.rollbackBlueGreen(ctx, cfg, record)
@@ -272,10 +330,29 @@ func (m Manager) Rollback(ctx context.Context, cfg config.Config) (state.Record,
err = errors.New("unsupported strategy")
}
if err != nil {
emit("failed")
return state.Record{}, err
}
emit("succeeded")
return state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
}
func releaseDigest(release string) (string, error) {
name := filepath.Base(release)
if !strings.HasPrefix(name, "sha256-") {
return "", errors.New("release is not content addressed")
}
digest := strings.TrimPrefix(name, "sha256-")
if len(digest) != 64 {
return "", errors.New("release digest is invalid")
}
for _, character := range digest {
if !strings.ContainsRune("0123456789abcdef", character) {
return "", errors.New("release digest is invalid")
}
}
return digest, nil
}
func (m Manager) rollbackBlueGreen(ctx context.Context, cfg config.Config, record state.Record) (err error) {
bg := *cfg.Deployment.BlueGreen
slot := slotConfig(bg, record.PreviousSlot)
@@ -321,36 +398,148 @@ func (m Manager) rollbackBlueGreen(ctx context.Context, cfg config.Config, recor
if err = m.probePublic(ctx, cfg, false); err != nil {
return err
}
next := state.Record{SchemaVersion: 1, Strategy: record.Strategy, ActiveSlot: record.PreviousSlot, ActiveRelease: record.PreviousRelease, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
next := state.Record{SchemaVersion: 1, Strategy: record.Strategy, DesiredRelease: record.PreviousRelease, ActiveSlot: record.PreviousSlot, ActiveRelease: record.PreviousRelease, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, LastAttemptRelease: record.PreviousRelease, LastAttemptOutcome: "rolled_back", LastAttemptAt: m.Now().UTC().Format(time.RFC3339), UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next)
}
func (m Manager) rollbackSingleton(ctx context.Context, cfg config.Config, record state.Record) (err error) {
if err = m.activateSingletonRelease(ctx, cfg, record.PreviousRelease, false); err != nil {
return err
}
next := state.Record{SchemaVersion: 1, Strategy: record.Strategy, DesiredRelease: record.PreviousRelease, ActiveSlot: "singleton", ActiveRelease: record.PreviousRelease, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, LastAttemptRelease: record.PreviousRelease, LastAttemptOutcome: "rolled_back", LastAttemptAt: m.Now().UTC().Format(time.RFC3339), UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next)
}
// activateSingletonRelease keeps public traffic on a proven process while the
// installed fixed-address unit changes release. The transient candidate first
// receives traffic, remains healthy through the handoff, and is stopped only
// after Caddy points back to the verified installed unit.
func (m Manager) activateSingletonRelease(ctx context.Context, cfg config.Config, release string, checkMarkers bool) (err error) {
single := *cfg.Deployment.Singleton
candidateUnit := cfg.Service.Name + "-tend-candidate.service"
env := map[string]string{single.ListenEnv: single.CandidateAddress}
binary := filepath.Join(release, cfg.Build.Binary)
if err = m.Operator.StartCandidate(ctx, candidateUnit, binary, cfg.Service.EnvironmentFile, env); err != nil {
return err
}
stopCandidate := true
defer func() {
if !stopCandidate {
return
}
stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
_ = m.Operator.Stop(stopCtx, candidateUnit)
}()
probeLocal := m.probeHealthReadiness
if checkMarkers {
probeLocal = m.probeAll
}
if err = probeLocal(ctx, cfg, single.CandidateAddress); err != nil {
return fmt.Errorf("candidate failed: %w", err)
}
oldHandler, err := os.ReadFile(single.CaddyHandler)
if err != nil {
return fmt.Errorf("read current Caddy handler: %w", err)
}
oldCurrent, err := resolveReleaseLink(cfg.Deployment.Root, single.CurrentLink)
if err != nil {
return err
}
if err = replaceSymlink(single.CurrentLink, record.PreviousRelease); err != nil {
return err
}
oldPrevious, previousErr := resolveReleaseLink(cfg.Deployment.Root, single.PreviousLink)
handlerChanged := false
currentChanged := false
previousChanged := false
defer func() {
if err != nil {
_ = replaceSymlink(single.CurrentLink, oldCurrent)
_ = m.Operator.Restart(ctx, single.Unit)
if err == nil {
return
}
recoveryErr := error(nil)
if currentChanged {
if restoreErr := replaceSymlink(single.CurrentLink, oldCurrent); restoreErr != nil {
recoveryErr = errors.Join(recoveryErr, restoreErr)
} else if restoreErr = m.Operator.Restart(ctx, single.Unit); restoreErr != nil {
recoveryErr = errors.Join(recoveryErr, restoreErr)
} else if restoreErr = m.probeHealthReadiness(ctx, cfg, single.Address); restoreErr != nil {
recoveryErr = errors.Join(recoveryErr, restoreErr)
}
}
if previousChanged {
var restoreErr error
if previousErr == nil {
restoreErr = replaceSymlink(single.PreviousLink, oldPrevious)
} else {
restoreErr = removeSymlink(single.PreviousLink)
}
recoveryErr = errors.Join(recoveryErr, restoreErr)
}
if handlerChanged && recoveryErr == nil {
if restoreErr := atomicWrite(single.CaddyHandler, oldHandler, 0o644); restoreErr != nil {
recoveryErr = errors.Join(recoveryErr, restoreErr)
} else if restoreErr = m.Operator.ValidateCaddy(ctx, single.CaddyConfig); restoreErr != nil {
recoveryErr = errors.Join(recoveryErr, restoreErr)
} else if restoreErr = m.Operator.ReloadCaddy(ctx); restoreErr != nil {
recoveryErr = errors.Join(recoveryErr, restoreErr)
}
}
if recoveryErr != nil && handlerChanged {
stopCandidate = false
err = errors.Join(err, fmt.Errorf("singleton recovery incomplete; candidate remains routed for operator recovery: %w", recoveryErr))
}
}()
candidateHandler, err := renderHandler(single.CaddyHandlerTemplate, single.CandidateAddress)
if err != nil {
return err
}
if err = atomicWrite(single.CaddyHandler, candidateHandler, 0o644); err != nil {
return err
}
handlerChanged = true
if err = m.Operator.ValidateCaddy(ctx, single.CaddyConfig); err != nil {
return fmt.Errorf("candidate Caddy validation failed: %w", err)
}
if err = m.Operator.ReloadCaddy(ctx); err != nil {
return fmt.Errorf("candidate Caddy reload failed: %w", err)
}
if err = m.probePublic(ctx, cfg, checkMarkers); err != nil {
return fmt.Errorf("candidate public-origin smoke failed: %w", err)
}
if err = replaceSymlink(single.PreviousLink, oldCurrent); err != nil {
return err
}
previousChanged = true
if err = replaceSymlink(single.CurrentLink, release); err != nil {
return err
}
currentChanged = true
if err = m.Operator.Restart(ctx, single.Unit); err != nil {
return err
}
if err = m.probeHealthReadiness(ctx, cfg, single.Address); err != nil {
if err = probeLocal(ctx, cfg, single.Address); err != nil {
return fmt.Errorf("post-activation smoke failed: %w", err)
}
installedHandler, err := renderHandler(single.CaddyHandlerTemplate, single.Address)
if err != nil {
return err
}
if err = m.probePublic(ctx, cfg, false); err != nil {
if err = atomicWrite(single.CaddyHandler, installedHandler, 0o644); err != nil {
return err
}
_ = replaceSymlink(single.PreviousLink, record.ActiveRelease)
next := state.Record{SchemaVersion: 1, Strategy: record.Strategy, ActiveSlot: "singleton", ActiveRelease: record.PreviousRelease, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next)
if err = m.Operator.ValidateCaddy(ctx, single.CaddyConfig); err != nil {
return fmt.Errorf("installed Caddy validation failed: %w", err)
}
if err = m.Operator.ReloadCaddy(ctx); err != nil {
return fmt.Errorf("installed Caddy reload failed: %w", err)
}
if err = m.probePublic(ctx, cfg, checkMarkers); err != nil {
return fmt.Errorf("public-origin smoke failed: %w", err)
}
if err = m.continuityWindow(ctx, cfg, single.CandidateAddress, checkMarkers); err != nil {
return fmt.Errorf("activation continuity failed: %w", err)
}
return nil
}
func (m Manager) Status(ctx context.Context, cfg config.Config) (Status, error) {
@@ -463,6 +652,33 @@ func (m Manager) probePublic(ctx context.Context, cfg config.Config, checkMarker
return nil
}
func (m Manager) continuityWindow(ctx context.Context, cfg config.Config, previousAddress string, checkMarkers bool) error {
steps := cfg.Deployment.ActivationWindowSecs * 4
if steps < 1 {
steps = 1
}
for step := 0; step < steps; step++ {
if err := m.probePublic(ctx, cfg, checkMarkers); err != nil {
return err
}
if previousAddress != "" {
if err := m.probeHealthReadiness(ctx, cfg, previousAddress); err != nil {
return fmt.Errorf("previous slot lost continuity: %w", err)
}
}
if step+1 < steps {
sleep := m.Sleep
if sleep == nil {
sleep = sleepContext
}
if err := sleep(ctx, 250*time.Millisecond); err != nil {
return err
}
}
}
return nil
}
func (m Manager) probe(ctx context.Context, cfg config.Config, address string, checks []config.Smoke) error {
timeout := time.Duration(cfg.Deployment.CandidateTimeoutSecs) * time.Second
for _, check := range checks {
+233 -22
View File
@@ -5,6 +5,7 @@ package deploy
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
@@ -12,18 +13,23 @@ import (
"time"
"gamertan.com/tend/internal/config"
"gamertan.com/tend/internal/eventlog"
"gamertan.com/tend/internal/state"
)
type fakeOperator struct {
failReload bool
failReloadAt int
reloads int
failRestartUnit string
failRestartOnce bool
rejectMarkers bool
active map[string]bool
starts, stops, restarts []string
probes []string
publicProbes []string
failPublic bool
failPublicAfter int
candidateEnvironment map[string]string
candidateFile string
}
@@ -31,6 +37,9 @@ type fakeOperator struct {
func (f *fakeOperator) Restart(_ context.Context, unit string) error {
f.restarts = append(f.restarts, unit)
if unit == f.failRestartUnit {
if f.failRestartOnce {
f.failRestartUnit = ""
}
return errors.New("injected restart failure")
}
f.active[unit] = true
@@ -59,7 +68,7 @@ func (f *fakeOperator) StartCandidate(_ context.Context, unit, binary, environme
}
func (f *fakeOperator) ProbeURL(_ context.Context, value, contains string) error {
f.publicProbes = append(f.publicProbes, value)
if f.failPublic {
if f.failPublic || (f.failPublicAfter > 0 && len(f.publicProbes) >= f.failPublicAfter) {
return errors.New("injected public smoke failure")
}
if f.rejectMarkers && contains != "" {
@@ -92,13 +101,44 @@ func TestPublicSmokeFailureRestoresBlueGreenHandlerAndSlot(t *testing.T) {
if err != nil || target != old {
t.Fatalf("green=%q err=%v", target, err)
}
if _, err = os.Stat(cfg.Deployment.StateFile); !os.IsNotExist(err) {
t.Fatal("failed public smoke wrote state")
record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
if err != nil {
t.Fatal(err)
}
if record.ActiveRelease != old || record.LastAttemptOutcome != "failed" || record.CandidateRelease != "" || record.LastAttemptRelease != fresh {
t.Fatalf("failed attempt state=%+v", record)
}
}
func TestContinuityFailureRestoresBlueGreenRoute(t *testing.T) {
cfg, old, fresh := baseConfig(t, "blue_green")
handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
template := filepath.Join(cfg.Deployment.Root, "handler.template")
original := []byte("reverse_proxy 127.0.0.1:8090\n")
_ = os.WriteFile(handler, original, 0o644)
_ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644)
blue := filepath.Join(cfg.Deployment.Root, "slots", "blue")
green := filepath.Join(cfg.Deployment.Root, "slots", "green")
_ = replaceSymlink(blue, old)
_ = replaceSymlink(green, old)
cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}}
operator := &fakeOperator{active: map[string]bool{}, failPublicAfter: 3}
if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err == nil || !strings.Contains(err.Error(), "continuity") {
t.Fatalf("expected continuity failure, got %v", err)
}
body, _ := os.ReadFile(handler)
if string(body) != string(original) {
t.Fatalf("handler not restored: %q", body)
}
target, err := resolveReleaseLink(cfg.Deployment.Root, green)
if err != nil || target != old {
t.Fatalf("green=%q err=%v", target, err)
}
}
func (f *fakeOperator) ValidateCaddy(context.Context, string) error { return nil }
func (f *fakeOperator) ReloadCaddy(context.Context) error {
if f.failReload {
f.reloads++
if f.failReload || (f.failReloadAt > 0 && f.reloads == f.failReloadAt) {
return errors.New("injected reload failure")
}
return nil
@@ -117,7 +157,7 @@ func baseConfig(t *testing.T, strategy string) (config.Config, string, string) {
if err := os.MkdirAll(filepath.Join(root, "releases"), 0o755); err != nil {
t.Fatal(err)
}
old := filepath.Join(root, "releases", "legacy-old")
old := filepath.Join(root, "releases", "sha256-"+strings.Repeat("c", 64))
fresh := filepath.Join(root, "releases", "sha256-"+strings.Repeat("a", 64))
for _, dir := range []string{old, fresh} {
if err := os.Mkdir(dir, 0o755); err != nil {
@@ -127,11 +167,34 @@ func baseConfig(t *testing.T, strategy string) (config.Config, string, string) {
t.Fatal(err)
}
}
cfg := config.Config{SchemaVersion: 2, Service: config.Service{Name: "example-site", AllowedHost: "example.test", EnvironmentFile: "/etc/tend/environment/example-site.env"}, Build: config.Build{Package: "./cmd/site", Binary: "app", Branch: "main"}, Deployment: config.Deployment{Strategy: strategy, Root: root, LockFile: filepath.Join(root, "deploy.lock"), StateFile: filepath.Join(root, "state.json"), HealthPath: "/healthz", ReadinessPath: "/readyz", CandidateTimeoutSecs: 2, Smoke: []config.Smoke{{Path: "/", Contains: "Example"}}, PublicSmoke: []config.PublicSmoke{{URL: "https://example.test/", Contains: "Example"}}}}
cfg := config.Config{SchemaVersion: 2, Service: config.Service{Name: "example-site", AllowedHost: "example.test", EnvironmentFile: "/etc/tend/environment/example-site.env"}, Build: config.Build{Package: "./cmd/site", Binary: "app", Branch: "main"}, Deployment: config.Deployment{Strategy: strategy, Root: root, LockFile: filepath.Join(root, "deploy.lock"), StateFile: filepath.Join(root, "state.json"), EventLog: filepath.Join(root, "deployment-events.jsonl"), HealthPath: "/healthz", ReadinessPath: "/readyz", CandidateTimeoutSecs: 2, ActivationWindowSecs: 1, Smoke: []config.Smoke{{Path: "/", Contains: "Example"}}, PublicSmoke: []config.PublicSmoke{{URL: "https://example.test/", Contains: "Example"}}}}
return cfg, old, fresh
}
func manager(operator Operator, fresh string) Manager {
return Manager{Operator: operator, Now: func() time.Time { return time.Unix(100, 0).UTC() }, Prepare: func(config.Config, string, string, string) (string, error) { return fresh, nil }, Inspect: func(config.Config, string, string, string) error { return nil }}
return Manager{Operator: operator, Now: func() time.Time { return time.Unix(100, 0).UTC() }, Prepare: func(config.Config, string, string, string) (string, error) { return fresh, nil }, Inspect: func(config.Config, string, string, string) error { return nil }, ReadIdentity: func(string) (releaseIdentity, error) {
return releaseIdentity{Version: "v0.2.0-preview.1", Commit: strings.Repeat("b", 40)}, nil
}, OperationID: func() (string, error) { return strings.Repeat("d", 32), nil }, Sleep: func(context.Context, time.Duration) error { return nil }}
}
func singletonSettings(t *testing.T, cfg config.Config, currentRelease string) (*config.Singleton, string) {
t.Helper()
handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
template := filepath.Join(cfg.Deployment.Root, "handler.template")
if err := os.WriteFile(handler, []byte("reverse_proxy 127.0.0.1:8092\n"), 0o640); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644); err != nil {
t.Fatal(err)
}
current := filepath.Join(cfg.Deployment.Root, "current")
if err := replaceSymlink(current, currentRelease); err != nil {
t.Fatal(err)
}
return &config.Singleton{
Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN",
CurrentLink: current, PreviousLink: filepath.Join(cfg.Deployment.Root, "previous"), CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"),
CaddyHandler: handler, CaddyHandlerTemplate: template,
}, handler
}
func TestBlueGreenActivationAndRollback(t *testing.T) {
@@ -158,13 +221,28 @@ func TestBlueGreenActivationAndRollback(t *testing.T) {
}
operator := &fakeOperator{active: map[string]bool{"example-blue.service": true, "example-green.service": true}}
m := manager(operator, fresh)
report, err := m.Deploy(context.Background(), cfg, Request{Activate: true})
var events []eventlog.Event
m.AppendEvent = func(_ string, event eventlog.Event) error {
events = append(events, event)
return nil
}
report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
if err != nil {
t.Fatal(err)
}
if report.ActiveRelease != fresh || report.PreviousRelease != old {
t.Fatalf("report=%+v", report)
}
deployed, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
if err != nil {
t.Fatal(err)
}
if deployed.DesiredRelease != fresh || deployed.CandidateRelease != "" || deployed.LastAttemptRelease != fresh || deployed.LastAttemptOutcome != "succeeded" {
t.Fatalf("deployment identity state=%+v", deployed)
}
if len(events) != 2 || events[0].Phase != "candidate" || events[0].Outcome != "running" || events[1].Phase != "activation" || events[1].Outcome != "succeeded" || events[0].OperationID != events[1].OperationID {
t.Fatalf("events=%+v", events)
}
if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 {
t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err)
}
@@ -180,6 +258,9 @@ func TestBlueGreenActivationAndRollback(t *testing.T) {
if record.ActiveRelease != old || record.PreviousRelease != fresh {
t.Fatalf("rollback=%+v", record)
}
if len(events) != 4 || events[2].Phase != "rollback" || events[2].Outcome != "running" || events[3].Phase != "rollback" || events[3].Outcome != "succeeded" || events[2].OperationID != events[3].OperationID || events[2].ArtifactDigest != strings.Repeat("c", 64) {
t.Fatalf("rollback events=%+v", events)
}
if len(operator.probes) != 4 {
t.Fatalf("rollback probes=%#v", operator.probes)
}
@@ -190,6 +271,58 @@ func TestBlueGreenActivationAndRollback(t *testing.T) {
}
}
func TestDeploymentEvidenceCanNeverBlockActivationOrRollback(t *testing.T) {
cfg, old, fresh := baseConfig(t, "singleton_candidate")
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
m := manager(&fakeOperator{active: map[string]bool{}}, fresh)
appendCalls := 0
m.AppendEvent = func(string, eventlog.Event) error {
appendCalls++
return errors.New("injected event failure")
}
report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
if err != nil {
t.Fatal(err)
}
if report.EventWarnings != 2 || report.ActiveRelease != fresh {
t.Fatalf("report=%+v", report)
}
if _, err := m.Rollback(context.Background(), cfg); err != nil {
t.Fatal(err)
}
record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
if err != nil || record.ActiveRelease != old || appendCalls != 4 {
t.Fatalf("record=%+v appends=%d err=%v", record, appendCalls, err)
}
}
func TestDeploymentEvidenceIdentityAndEntropyAreBestEffort(t *testing.T) {
for _, test := range []struct {
name string
damage func(*Manager)
}{
{"identity", func(manager *Manager) {
manager.ReadIdentity = func(string) (releaseIdentity, error) {
return releaseIdentity{}, errors.New("injected identity failure")
}
}},
{"entropy", func(manager *Manager) {
manager.OperationID = func() (string, error) { return "", errors.New("injected entropy failure") }
}},
} {
t.Run(test.name, func(t *testing.T) {
cfg, old, fresh := baseConfig(t, "singleton_candidate")
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
manager := manager(&fakeOperator{active: map[string]bool{}}, fresh)
test.damage(&manager)
report, err := manager.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
if err != nil || report.EventWarnings != 1 || report.ActiveRelease != fresh {
t.Fatalf("report=%+v err=%v", report, err)
}
})
}
}
func TestBlueGreenCaddyFailureRestoresHandlerAndSlot(t *testing.T) {
cfg, old, fresh := baseConfig(t, "blue_green")
handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
@@ -215,37 +348,47 @@ func TestBlueGreenCaddyFailureRestoresHandlerAndSlot(t *testing.T) {
if err != nil || target != old {
t.Fatalf("green=%q err=%v", target, err)
}
if _, err := os.Stat(cfg.Deployment.StateFile); !os.IsNotExist(err) {
t.Fatal("failed activation wrote state")
failed, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
if err != nil {
t.Fatal(err)
}
if failed.ActiveRelease != old || failed.LastAttemptOutcome != "failed" || failed.CandidateRelease != "" {
t.Fatalf("failed state=%+v", failed)
}
}
func TestSingletonRestartFailureRestoresPointers(t *testing.T) {
cfg, old, fresh := baseConfig(t, "singleton_candidate")
previous := filepath.Join(cfg.Deployment.Root, "previous")
current := filepath.Join(cfg.Deployment.Root, "current")
_ = replaceSymlink(current, old)
cfg.Deployment.Singleton = &config.Singleton{Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN", CurrentLink: current, PreviousLink: previous}
operator := &fakeOperator{active: map[string]bool{}, failRestartUnit: "example-site.service"}
settings, handler := singletonSettings(t, cfg, old)
cfg.Deployment.Singleton = settings
operator := &fakeOperator{active: map[string]bool{}, failRestartUnit: "example-site.service", failRestartOnce: true}
m := manager(operator, fresh)
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil {
t.Fatal("expected failure")
}
target, err := resolveReleaseLink(cfg.Deployment.Root, current)
target, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink)
if err != nil || target != old {
t.Fatalf("current=%q err=%v", target, err)
}
if _, err := os.Lstat(previous); !os.IsNotExist(err) {
if _, err := os.Lstat(cfg.Deployment.Singleton.PreviousLink); !os.IsNotExist(err) {
t.Fatal("previous pointer was not restored")
}
body, err := os.ReadFile(handler)
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
t.Fatalf("handler=%q err=%v", body, err)
}
if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 {
t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err)
}
if operator.active["example-site-tend-candidate.service"] {
t.Fatal("candidate was not stopped after successful restoration")
}
}
func TestStatePersistsOnlyAfterSuccessfulActivation(t *testing.T) {
func TestStateRecordsSuccessfulActivation(t *testing.T) {
cfg, old, fresh := baseConfig(t, "singleton_candidate")
current := filepath.Join(cfg.Deployment.Root, "current")
previous := filepath.Join(cfg.Deployment.Root, "previous")
_ = replaceSymlink(current, old)
cfg.Deployment.Singleton = &config.Singleton{Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN", CurrentLink: current, PreviousLink: previous}
settings, handler := singletonSettings(t, cfg, old)
cfg.Deployment.Singleton = settings
operator := &fakeOperator{active: map[string]bool{}}
m := manager(operator, fresh)
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err != nil {
@@ -261,4 +404,72 @@ func TestStatePersistsOnlyAfterSuccessfulActivation(t *testing.T) {
if operator.candidateFile != cfg.Service.EnvironmentFile || len(operator.candidateEnvironment) != 1 || operator.candidateEnvironment["EXAMPLE_LISTEN"] != "127.0.0.1:18092" {
t.Fatalf("candidate file=%q environment=%#v", operator.candidateFile, operator.candidateEnvironment)
}
body, err := os.ReadFile(handler)
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
t.Fatalf("handler=%q err=%v", body, err)
}
if operator.reloads != 2 || operator.active["example-site-tend-candidate.service"] {
t.Fatalf("reloads=%d active=%#v", operator.reloads, operator.active)
}
}
func TestSingletonContinuityFailureRestoresHandlerPointersAndService(t *testing.T) {
cfg, old, fresh := baseConfig(t, "singleton_candidate")
settings, handler := singletonSettings(t, cfg, old)
cfg.Deployment.Singleton = settings
operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failPublicAfter: 4}
m := manager(operator, fresh)
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil || !strings.Contains(err.Error(), "continuity") {
t.Fatalf("expected continuity failure, got %v", err)
}
current, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink)
if err != nil || current != old {
t.Fatalf("current=%q err=%v", current, err)
}
body, err := os.ReadFile(handler)
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
t.Fatalf("handler=%q err=%v", body, err)
}
if operator.active["example-site-tend-candidate.service"] {
t.Fatal("candidate was not stopped after continuity restoration")
}
}
func TestSingletonCaddyReloadFailuresRestorePriorRoute(t *testing.T) {
for _, reload := range []int{1, 2} {
t.Run(fmt.Sprintf("reload-%d", reload), func(t *testing.T) {
cfg, old, fresh := baseConfig(t, "singleton_candidate")
settings, handler := singletonSettings(t, cfg, old)
cfg.Deployment.Singleton = settings
operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failReloadAt: reload}
if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true}); err == nil || !strings.Contains(err.Error(), "Caddy reload failed") {
t.Fatalf("expected Caddy reload failure, got %v", err)
}
current, err := resolveReleaseLink(cfg.Deployment.Root, settings.CurrentLink)
if err != nil || current != old {
t.Fatalf("current=%q err=%v", current, err)
}
body, err := os.ReadFile(handler)
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
t.Fatalf("handler=%q err=%v", body, err)
}
if operator.active["example-site-tend-candidate.service"] {
t.Fatal("candidate was not stopped after route restoration")
}
})
}
}
func TestSingletonIncompleteRecoveryKeepsProvenCandidateRunning(t *testing.T) {
cfg, old, fresh := baseConfig(t, "singleton_candidate")
settings, _ := singletonSettings(t, cfg, old)
cfg.Deployment.Singleton = settings
operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failReload: true}
_, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true})
if err == nil || !strings.Contains(err.Error(), "candidate remains routed for operator recovery") {
t.Fatalf("expected explicit incomplete recovery, got %v", err)
}
if !operator.active["example-site-tend-candidate.service"] {
t.Fatal("proven candidate was stopped despite incomplete route restoration")
}
}
+9
View File
@@ -196,6 +196,15 @@ func extractArtifact(artifact, stage string) error {
_ = out.Close()
return err
}
// OpenFile modes are filtered through the caller's umask. Tend is
// commonly invoked by a root account with umask 0077, while release
// binaries must remain executable by their dedicated service users.
// Reapply the validated, name-derived mode explicitly before the file
// becomes part of an immutable release.
if err := out.Chmod(mode); err != nil {
_ = out.Close()
return err
}
if err := out.Sync(); err != nil {
_ = out.Close()
return err
@@ -0,0 +1,75 @@
// SPDX-License-Identifier: AGPL-3.0-only
//go:build linux
package deploy
import (
"archive/tar"
"compress/gzip"
"os"
"path/filepath"
"syscall"
"testing"
)
func TestExtractArtifactAppliesReleaseModesUnderRestrictiveUmask(t *testing.T) {
dir := t.TempDir()
artifact := filepath.Join(dir, "release.tar.gz")
file, err := os.OpenFile(artifact, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
if err != nil {
t.Fatal(err)
}
gz := gzip.NewWriter(file)
tw := tar.NewWriter(gz)
entries := map[string][]byte{
"bundle/app": []byte("executable"),
"bundle/BUILDINFO.json": []byte("{}"),
"bundle/RELEASE.json": []byte("{}"),
"bundle/SBOM.spdx.json": []byte("{}"),
"bundle/SHA256SUMS": []byte("checksums"),
}
for name, body := range entries {
if err := tw.WriteHeader(&tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0o600, Size: int64(len(body))}); err != nil {
t.Fatal(err)
}
if _, err := tw.Write(body); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
if err := gz.Close(); err != nil {
t.Fatal(err)
}
if err := file.Close(); err != nil {
t.Fatal(err)
}
oldUmask := syscall.Umask(0o077)
t.Cleanup(func() { syscall.Umask(oldUmask) })
stage := filepath.Join(dir, "stage")
if err := os.Mkdir(stage, 0o700); err != nil {
t.Fatal(err)
}
if err := extractArtifact(artifact, stage); err != nil {
t.Fatal(err)
}
for name, want := range map[string]os.FileMode{
"app": 0o755,
"BUILDINFO.json": 0o644,
"RELEASE.json": 0o644,
"SBOM.spdx.json": 0o644,
"SHA256SUMS": 0o644,
} {
info, err := os.Stat(filepath.Join(stage, name))
if err != nil {
t.Fatal(err)
}
if got := info.Mode().Perm(); got != want {
t.Fatalf("%s mode=%#o want=%#o", name, got, want)
}
}
}