feat: publish Tend v0.2 Preview 2 source
Export the reviewed allowlisted snapshot from private source commit 8aab3db43f35e6a49aa497f45d73701b13fc9f32 and tree 992132ea4703437dc13ffdbb04a077816c02caf9. This includes routed singleton continuity, deployment evidence, strict schema-2 configuration, restricted transport, and the independently compilable public-tree guard. AI-Assisted: OpenAI Codex Signed-off-by: Cole Speelman <crspeelman@gmail.com>
This commit is contained in:
+292
-76
@@ -14,6 +14,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gamertan.com/tend/internal/config"
|
||||
"gamertan.com/tend/internal/eventlog"
|
||||
"gamertan.com/tend/internal/state"
|
||||
)
|
||||
|
||||
@@ -29,6 +30,7 @@ type Report struct {
|
||||
Release string `json:"release,omitempty"`
|
||||
ActiveRelease string `json:"active_release,omitempty"`
|
||||
PreviousRelease string `json:"previous_release,omitempty"`
|
||||
EventWarnings int `json:"event_warnings,omitempty"`
|
||||
}
|
||||
type Status struct {
|
||||
State *state.Record `json:"state,omitempty"`
|
||||
@@ -37,14 +39,18 @@ type Status struct {
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
Operator Operator
|
||||
Now func() time.Time
|
||||
Prepare func(config.Config, string, string, string) (string, error)
|
||||
Inspect func(config.Config, string, string, string) error
|
||||
Operator Operator
|
||||
Now func() time.Time
|
||||
Prepare func(config.Config, string, string, string) (string, error)
|
||||
Inspect func(config.Config, string, string, string) error
|
||||
ReadIdentity func(string) (releaseIdentity, error)
|
||||
OperationID func() (string, error)
|
||||
AppendEvent func(string, eventlog.Event) error
|
||||
Sleep func(context.Context, time.Duration) error
|
||||
}
|
||||
|
||||
func NewManager(operator Operator) Manager {
|
||||
return Manager{Operator: operator, Now: time.Now, Prepare: prepareRelease, Inspect: inspectArtifact}
|
||||
return Manager{Operator: operator, Now: time.Now, Prepare: prepareRelease, Inspect: inspectArtifact, ReadIdentity: readReleaseIdentity, OperationID: eventlog.OperationID, AppendEvent: eventlog.Append, Sleep: sleepContext}
|
||||
}
|
||||
|
||||
func (m Manager) Deploy(ctx context.Context, cfg config.Config, request Request) (Report, error) {
|
||||
@@ -66,10 +72,44 @@ func (m Manager) Deploy(ctx context.Context, cfg config.Config, request Request)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
started := m.Now()
|
||||
eventWarnings := 0
|
||||
identity, identityErr := m.ReadIdentity(release)
|
||||
if identityErr != nil {
|
||||
eventWarnings++
|
||||
}
|
||||
operationID := ""
|
||||
if m.OperationID != nil {
|
||||
operationID, err = m.OperationID()
|
||||
if err != nil {
|
||||
eventWarnings++
|
||||
operationID = ""
|
||||
}
|
||||
}
|
||||
emit := func(phase, slot, outcome string) {
|
||||
if m.AppendEvent == nil || identityErr != nil || operationID == "" {
|
||||
return
|
||||
}
|
||||
event := eventlog.Event{Version: eventlog.Version, OperationID: operationID, Service: cfg.Service.Name, ArtifactDigest: request.ApprovedSHA256, Commit: identity.Commit, ReleaseVersion: identity.Version, Phase: phase, Slot: slot, DurationMillis: max(0, m.Now().Sub(started).Milliseconds()), Outcome: outcome, ObservedAt: m.Now().UTC().Format(time.RFC3339Nano)}
|
||||
if eventErr := m.AppendEvent(cfg.Deployment.EventLog, event); eventErr != nil {
|
||||
eventWarnings++
|
||||
}
|
||||
}
|
||||
record, err := loadOrBootstrap(cfg, m.Now())
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
attemptAt := m.Now().UTC().Format(time.RFC3339)
|
||||
record.DesiredRelease = release
|
||||
record.CandidateRelease = release
|
||||
record.LastAttemptRelease = release
|
||||
record.LastAttemptOutcome = "running"
|
||||
record.LastAttemptAt = attemptAt
|
||||
record.UpdatedAt = attemptAt
|
||||
if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
emit("candidate", inactiveSlot(cfg, record), "running")
|
||||
switch cfg.Deployment.Strategy {
|
||||
case "blue_green":
|
||||
err = m.deployBlueGreen(ctx, cfg, record, release)
|
||||
@@ -79,13 +119,64 @@ func (m Manager) Deploy(ctx context.Context, cfg config.Config, request Request)
|
||||
err = errors.New("unsupported strategy")
|
||||
}
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
failed := record
|
||||
failed.CandidateRelease = ""
|
||||
failed.LastAttemptOutcome = "failed"
|
||||
failed.UpdatedAt = m.Now().UTC().Format(time.RFC3339)
|
||||
_ = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, failed)
|
||||
emit("activation", inactiveSlot(cfg, record), "failed")
|
||||
return Report{EventWarnings: eventWarnings}, err
|
||||
}
|
||||
emit("activation", inactiveSlot(cfg, record), "succeeded")
|
||||
updated, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
return Report{Validated: true, Mutation: "activated", Release: release, ActiveRelease: updated.ActiveRelease, PreviousRelease: updated.PreviousRelease}, nil
|
||||
return Report{Validated: true, Mutation: "activated", Release: release, ActiveRelease: updated.ActiveRelease, PreviousRelease: updated.PreviousRelease, EventWarnings: eventWarnings}, nil
|
||||
}
|
||||
|
||||
type releaseIdentity struct {
|
||||
Version string `json:"version"`
|
||||
Commit string `json:"commit"`
|
||||
}
|
||||
|
||||
func readReleaseIdentity(release string) (releaseIdentity, error) {
|
||||
b, err := os.ReadFile(filepath.Join(release, "RELEASE.json"))
|
||||
if err != nil {
|
||||
return releaseIdentity{}, fmt.Errorf("read installed release identity: %w", err)
|
||||
}
|
||||
if len(b) > 1<<20 {
|
||||
return releaseIdentity{}, errors.New("installed release identity is too large")
|
||||
}
|
||||
var identity releaseIdentity
|
||||
if err := json.Unmarshal(b, &identity); err != nil {
|
||||
return releaseIdentity{}, errors.New("decode installed release identity")
|
||||
}
|
||||
if identity.Version == "" || identity.Commit == "" {
|
||||
return releaseIdentity{}, errors.New("installed release identity is incomplete")
|
||||
}
|
||||
return identity, nil
|
||||
}
|
||||
|
||||
func inactiveSlot(cfg config.Config, record state.Record) string {
|
||||
if cfg.Deployment.Strategy == "singleton_candidate" {
|
||||
return "singleton"
|
||||
}
|
||||
if record.ActiveSlot == "blue" {
|
||||
return "green"
|
||||
}
|
||||
return "blue"
|
||||
}
|
||||
|
||||
func sleepContext(ctx context.Context, duration time.Duration) error {
|
||||
timer := time.NewTimer(duration)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func loadOrBootstrap(cfg config.Config, now time.Time) (state.Record, error) {
|
||||
@@ -103,13 +194,13 @@ func loadOrBootstrap(cfg config.Config, now time.Time) (state.Record, error) {
|
||||
if err != nil {
|
||||
return state.Record{}, fmt.Errorf("bootstrap active slot: %w", err)
|
||||
}
|
||||
return state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, ActiveSlot: slot, ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil
|
||||
return state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: slot, ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil
|
||||
case "singleton_candidate":
|
||||
release, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink)
|
||||
if err != nil {
|
||||
return state.Record{}, fmt.Errorf("bootstrap singleton: %w", err)
|
||||
}
|
||||
return state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, ActiveSlot: "singleton", ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil
|
||||
return state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil
|
||||
}
|
||||
return state.Record{}, errors.New("unsupported strategy")
|
||||
}
|
||||
@@ -180,7 +271,11 @@ func (m Manager) deployBlueGreen(ctx context.Context, cfg config.Config, record
|
||||
if err = m.probePublic(ctx, cfg, true); err != nil {
|
||||
return fmt.Errorf("public-origin smoke failed: %w", err)
|
||||
}
|
||||
next := state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, ActiveSlot: inactive, ActiveRelease: release, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
||||
previous := slotConfig(bg, record.ActiveSlot)
|
||||
if err = m.continuityWindow(ctx, cfg, previous.Address, true); err != nil {
|
||||
return fmt.Errorf("activation continuity failed: %w", err)
|
||||
}
|
||||
next := state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: inactive, ActiveRelease: release, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, LastAttemptRelease: release, LastAttemptOutcome: "succeeded", LastAttemptAt: record.LastAttemptAt, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
||||
if err = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -188,62 +283,10 @@ func (m Manager) deployBlueGreen(ctx context.Context, cfg config.Config, record
|
||||
}
|
||||
|
||||
func (m Manager) deploySingleton(ctx context.Context, cfg config.Config, record state.Record, release string) (err error) {
|
||||
single := *cfg.Deployment.Singleton
|
||||
candidateUnit := cfg.Service.Name + "-tend-candidate.service"
|
||||
env := map[string]string{single.ListenEnv: single.CandidateAddress}
|
||||
binary := filepath.Join(release, cfg.Build.Binary)
|
||||
if err = m.Operator.StartCandidate(ctx, candidateUnit, binary, cfg.Service.EnvironmentFile, env); err != nil {
|
||||
if err = m.activateSingletonRelease(ctx, cfg, release, true); err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = m.Operator.Stop(stopCtx, candidateUnit)
|
||||
}()
|
||||
if err = m.probeAll(ctx, cfg, single.CandidateAddress); err != nil {
|
||||
return fmt.Errorf("candidate failed: %w", err)
|
||||
}
|
||||
oldCurrent, err := resolveReleaseLink(cfg.Deployment.Root, single.CurrentLink)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
oldPrevious, previousErr := resolveReleaseLink(cfg.Deployment.Root, single.PreviousLink)
|
||||
currentChanged := false
|
||||
previousChanged := false
|
||||
defer func() {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
if currentChanged {
|
||||
_ = replaceSymlink(single.CurrentLink, oldCurrent)
|
||||
_ = m.Operator.Restart(ctx, single.Unit)
|
||||
}
|
||||
if previousChanged {
|
||||
if previousErr == nil {
|
||||
_ = replaceSymlink(single.PreviousLink, oldPrevious)
|
||||
} else {
|
||||
_ = removeSymlink(single.PreviousLink)
|
||||
}
|
||||
}
|
||||
}()
|
||||
if err = replaceSymlink(single.PreviousLink, oldCurrent); err != nil {
|
||||
return err
|
||||
}
|
||||
previousChanged = true
|
||||
if err = replaceSymlink(single.CurrentLink, release); err != nil {
|
||||
return err
|
||||
}
|
||||
currentChanged = true
|
||||
if err = m.Operator.Restart(ctx, single.Unit); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = m.probeAll(ctx, cfg, single.Address); err != nil {
|
||||
return fmt.Errorf("post-activation smoke failed: %w", err)
|
||||
}
|
||||
if err = m.probePublic(ctx, cfg, true); err != nil {
|
||||
return fmt.Errorf("public-origin smoke failed: %w", err)
|
||||
}
|
||||
next := state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, ActiveSlot: "singleton", ActiveRelease: release, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
||||
next := state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, LastAttemptRelease: release, LastAttemptOutcome: "succeeded", LastAttemptAt: record.LastAttemptAt, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
||||
if err = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -263,6 +306,21 @@ func (m Manager) Rollback(ctx context.Context, cfg config.Config) (state.Record,
|
||||
if record.PreviousRelease == "" {
|
||||
return state.Record{}, errors.New("no previous release is recorded")
|
||||
}
|
||||
started := m.Now()
|
||||
identity, identityErr := m.ReadIdentity(record.PreviousRelease)
|
||||
digest, digestErr := releaseDigest(record.PreviousRelease)
|
||||
operationID := ""
|
||||
if m.OperationID != nil {
|
||||
operationID, _ = m.OperationID()
|
||||
}
|
||||
emit := func(outcome string) {
|
||||
if m.AppendEvent == nil || identityErr != nil || digestErr != nil || operationID == "" {
|
||||
return
|
||||
}
|
||||
event := eventlog.Event{Version: eventlog.Version, OperationID: operationID, Service: cfg.Service.Name, ArtifactDigest: digest, Commit: identity.Commit, ReleaseVersion: identity.Version, Phase: "rollback", Slot: record.PreviousSlot, DurationMillis: max(0, m.Now().Sub(started).Milliseconds()), Outcome: outcome, ObservedAt: m.Now().UTC().Format(time.RFC3339Nano)}
|
||||
_ = m.AppendEvent(cfg.Deployment.EventLog, event)
|
||||
}
|
||||
emit("running")
|
||||
switch cfg.Deployment.Strategy {
|
||||
case "blue_green":
|
||||
err = m.rollbackBlueGreen(ctx, cfg, record)
|
||||
@@ -272,10 +330,29 @@ func (m Manager) Rollback(ctx context.Context, cfg config.Config) (state.Record,
|
||||
err = errors.New("unsupported strategy")
|
||||
}
|
||||
if err != nil {
|
||||
emit("failed")
|
||||
return state.Record{}, err
|
||||
}
|
||||
emit("succeeded")
|
||||
return state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
}
|
||||
|
||||
func releaseDigest(release string) (string, error) {
|
||||
name := filepath.Base(release)
|
||||
if !strings.HasPrefix(name, "sha256-") {
|
||||
return "", errors.New("release is not content addressed")
|
||||
}
|
||||
digest := strings.TrimPrefix(name, "sha256-")
|
||||
if len(digest) != 64 {
|
||||
return "", errors.New("release digest is invalid")
|
||||
}
|
||||
for _, character := range digest {
|
||||
if !strings.ContainsRune("0123456789abcdef", character) {
|
||||
return "", errors.New("release digest is invalid")
|
||||
}
|
||||
}
|
||||
return digest, nil
|
||||
}
|
||||
func (m Manager) rollbackBlueGreen(ctx context.Context, cfg config.Config, record state.Record) (err error) {
|
||||
bg := *cfg.Deployment.BlueGreen
|
||||
slot := slotConfig(bg, record.PreviousSlot)
|
||||
@@ -321,36 +398,148 @@ func (m Manager) rollbackBlueGreen(ctx context.Context, cfg config.Config, recor
|
||||
if err = m.probePublic(ctx, cfg, false); err != nil {
|
||||
return err
|
||||
}
|
||||
next := state.Record{SchemaVersion: 1, Strategy: record.Strategy, ActiveSlot: record.PreviousSlot, ActiveRelease: record.PreviousRelease, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
||||
next := state.Record{SchemaVersion: 1, Strategy: record.Strategy, DesiredRelease: record.PreviousRelease, ActiveSlot: record.PreviousSlot, ActiveRelease: record.PreviousRelease, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, LastAttemptRelease: record.PreviousRelease, LastAttemptOutcome: "rolled_back", LastAttemptAt: m.Now().UTC().Format(time.RFC3339), UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
||||
return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next)
|
||||
}
|
||||
func (m Manager) rollbackSingleton(ctx context.Context, cfg config.Config, record state.Record) (err error) {
|
||||
if err = m.activateSingletonRelease(ctx, cfg, record.PreviousRelease, false); err != nil {
|
||||
return err
|
||||
}
|
||||
next := state.Record{SchemaVersion: 1, Strategy: record.Strategy, DesiredRelease: record.PreviousRelease, ActiveSlot: "singleton", ActiveRelease: record.PreviousRelease, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, LastAttemptRelease: record.PreviousRelease, LastAttemptOutcome: "rolled_back", LastAttemptAt: m.Now().UTC().Format(time.RFC3339), UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
||||
return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next)
|
||||
}
|
||||
|
||||
// activateSingletonRelease keeps public traffic on a proven process while the
|
||||
// installed fixed-address unit changes release. The transient candidate first
|
||||
// receives traffic, remains healthy through the handoff, and is stopped only
|
||||
// after Caddy points back to the verified installed unit.
|
||||
func (m Manager) activateSingletonRelease(ctx context.Context, cfg config.Config, release string, checkMarkers bool) (err error) {
|
||||
single := *cfg.Deployment.Singleton
|
||||
candidateUnit := cfg.Service.Name + "-tend-candidate.service"
|
||||
env := map[string]string{single.ListenEnv: single.CandidateAddress}
|
||||
binary := filepath.Join(release, cfg.Build.Binary)
|
||||
if err = m.Operator.StartCandidate(ctx, candidateUnit, binary, cfg.Service.EnvironmentFile, env); err != nil {
|
||||
return err
|
||||
}
|
||||
stopCandidate := true
|
||||
defer func() {
|
||||
if !stopCandidate {
|
||||
return
|
||||
}
|
||||
stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = m.Operator.Stop(stopCtx, candidateUnit)
|
||||
}()
|
||||
probeLocal := m.probeHealthReadiness
|
||||
if checkMarkers {
|
||||
probeLocal = m.probeAll
|
||||
}
|
||||
if err = probeLocal(ctx, cfg, single.CandidateAddress); err != nil {
|
||||
return fmt.Errorf("candidate failed: %w", err)
|
||||
}
|
||||
|
||||
oldHandler, err := os.ReadFile(single.CaddyHandler)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read current Caddy handler: %w", err)
|
||||
}
|
||||
oldCurrent, err := resolveReleaseLink(cfg.Deployment.Root, single.CurrentLink)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = replaceSymlink(single.CurrentLink, record.PreviousRelease); err != nil {
|
||||
return err
|
||||
}
|
||||
oldPrevious, previousErr := resolveReleaseLink(cfg.Deployment.Root, single.PreviousLink)
|
||||
handlerChanged := false
|
||||
currentChanged := false
|
||||
previousChanged := false
|
||||
defer func() {
|
||||
if err != nil {
|
||||
_ = replaceSymlink(single.CurrentLink, oldCurrent)
|
||||
_ = m.Operator.Restart(ctx, single.Unit)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
recoveryErr := error(nil)
|
||||
if currentChanged {
|
||||
if restoreErr := replaceSymlink(single.CurrentLink, oldCurrent); restoreErr != nil {
|
||||
recoveryErr = errors.Join(recoveryErr, restoreErr)
|
||||
} else if restoreErr = m.Operator.Restart(ctx, single.Unit); restoreErr != nil {
|
||||
recoveryErr = errors.Join(recoveryErr, restoreErr)
|
||||
} else if restoreErr = m.probeHealthReadiness(ctx, cfg, single.Address); restoreErr != nil {
|
||||
recoveryErr = errors.Join(recoveryErr, restoreErr)
|
||||
}
|
||||
}
|
||||
if previousChanged {
|
||||
var restoreErr error
|
||||
if previousErr == nil {
|
||||
restoreErr = replaceSymlink(single.PreviousLink, oldPrevious)
|
||||
} else {
|
||||
restoreErr = removeSymlink(single.PreviousLink)
|
||||
}
|
||||
recoveryErr = errors.Join(recoveryErr, restoreErr)
|
||||
}
|
||||
if handlerChanged && recoveryErr == nil {
|
||||
if restoreErr := atomicWrite(single.CaddyHandler, oldHandler, 0o644); restoreErr != nil {
|
||||
recoveryErr = errors.Join(recoveryErr, restoreErr)
|
||||
} else if restoreErr = m.Operator.ValidateCaddy(ctx, single.CaddyConfig); restoreErr != nil {
|
||||
recoveryErr = errors.Join(recoveryErr, restoreErr)
|
||||
} else if restoreErr = m.Operator.ReloadCaddy(ctx); restoreErr != nil {
|
||||
recoveryErr = errors.Join(recoveryErr, restoreErr)
|
||||
}
|
||||
}
|
||||
if recoveryErr != nil && handlerChanged {
|
||||
stopCandidate = false
|
||||
err = errors.Join(err, fmt.Errorf("singleton recovery incomplete; candidate remains routed for operator recovery: %w", recoveryErr))
|
||||
}
|
||||
}()
|
||||
|
||||
candidateHandler, err := renderHandler(single.CaddyHandlerTemplate, single.CandidateAddress)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = atomicWrite(single.CaddyHandler, candidateHandler, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
handlerChanged = true
|
||||
if err = m.Operator.ValidateCaddy(ctx, single.CaddyConfig); err != nil {
|
||||
return fmt.Errorf("candidate Caddy validation failed: %w", err)
|
||||
}
|
||||
if err = m.Operator.ReloadCaddy(ctx); err != nil {
|
||||
return fmt.Errorf("candidate Caddy reload failed: %w", err)
|
||||
}
|
||||
if err = m.probePublic(ctx, cfg, checkMarkers); err != nil {
|
||||
return fmt.Errorf("candidate public-origin smoke failed: %w", err)
|
||||
}
|
||||
|
||||
if err = replaceSymlink(single.PreviousLink, oldCurrent); err != nil {
|
||||
return err
|
||||
}
|
||||
previousChanged = true
|
||||
if err = replaceSymlink(single.CurrentLink, release); err != nil {
|
||||
return err
|
||||
}
|
||||
currentChanged = true
|
||||
if err = m.Operator.Restart(ctx, single.Unit); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = m.probeHealthReadiness(ctx, cfg, single.Address); err != nil {
|
||||
if err = probeLocal(ctx, cfg, single.Address); err != nil {
|
||||
return fmt.Errorf("post-activation smoke failed: %w", err)
|
||||
}
|
||||
installedHandler, err := renderHandler(single.CaddyHandlerTemplate, single.Address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = m.probePublic(ctx, cfg, false); err != nil {
|
||||
if err = atomicWrite(single.CaddyHandler, installedHandler, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = replaceSymlink(single.PreviousLink, record.ActiveRelease)
|
||||
next := state.Record{SchemaVersion: 1, Strategy: record.Strategy, ActiveSlot: "singleton", ActiveRelease: record.PreviousRelease, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
||||
return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next)
|
||||
if err = m.Operator.ValidateCaddy(ctx, single.CaddyConfig); err != nil {
|
||||
return fmt.Errorf("installed Caddy validation failed: %w", err)
|
||||
}
|
||||
if err = m.Operator.ReloadCaddy(ctx); err != nil {
|
||||
return fmt.Errorf("installed Caddy reload failed: %w", err)
|
||||
}
|
||||
if err = m.probePublic(ctx, cfg, checkMarkers); err != nil {
|
||||
return fmt.Errorf("public-origin smoke failed: %w", err)
|
||||
}
|
||||
if err = m.continuityWindow(ctx, cfg, single.CandidateAddress, checkMarkers); err != nil {
|
||||
return fmt.Errorf("activation continuity failed: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m Manager) Status(ctx context.Context, cfg config.Config) (Status, error) {
|
||||
@@ -463,6 +652,33 @@ func (m Manager) probePublic(ctx context.Context, cfg config.Config, checkMarker
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m Manager) continuityWindow(ctx context.Context, cfg config.Config, previousAddress string, checkMarkers bool) error {
|
||||
steps := cfg.Deployment.ActivationWindowSecs * 4
|
||||
if steps < 1 {
|
||||
steps = 1
|
||||
}
|
||||
for step := 0; step < steps; step++ {
|
||||
if err := m.probePublic(ctx, cfg, checkMarkers); err != nil {
|
||||
return err
|
||||
}
|
||||
if previousAddress != "" {
|
||||
if err := m.probeHealthReadiness(ctx, cfg, previousAddress); err != nil {
|
||||
return fmt.Errorf("previous slot lost continuity: %w", err)
|
||||
}
|
||||
}
|
||||
if step+1 < steps {
|
||||
sleep := m.Sleep
|
||||
if sleep == nil {
|
||||
sleep = sleepContext
|
||||
}
|
||||
if err := sleep(ctx, 250*time.Millisecond); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m Manager) probe(ctx context.Context, cfg config.Config, address string, checks []config.Smoke) error {
|
||||
timeout := time.Duration(cfg.Deployment.CandidateTimeoutSecs) * time.Second
|
||||
for _, check := range checks {
|
||||
|
||||
+233
-22
@@ -5,6 +5,7 @@ package deploy
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -12,18 +13,23 @@ import (
|
||||
"time"
|
||||
|
||||
"gamertan.com/tend/internal/config"
|
||||
"gamertan.com/tend/internal/eventlog"
|
||||
"gamertan.com/tend/internal/state"
|
||||
)
|
||||
|
||||
type fakeOperator struct {
|
||||
failReload bool
|
||||
failReloadAt int
|
||||
reloads int
|
||||
failRestartUnit string
|
||||
failRestartOnce bool
|
||||
rejectMarkers bool
|
||||
active map[string]bool
|
||||
starts, stops, restarts []string
|
||||
probes []string
|
||||
publicProbes []string
|
||||
failPublic bool
|
||||
failPublicAfter int
|
||||
candidateEnvironment map[string]string
|
||||
candidateFile string
|
||||
}
|
||||
@@ -31,6 +37,9 @@ type fakeOperator struct {
|
||||
func (f *fakeOperator) Restart(_ context.Context, unit string) error {
|
||||
f.restarts = append(f.restarts, unit)
|
||||
if unit == f.failRestartUnit {
|
||||
if f.failRestartOnce {
|
||||
f.failRestartUnit = ""
|
||||
}
|
||||
return errors.New("injected restart failure")
|
||||
}
|
||||
f.active[unit] = true
|
||||
@@ -59,7 +68,7 @@ func (f *fakeOperator) StartCandidate(_ context.Context, unit, binary, environme
|
||||
}
|
||||
func (f *fakeOperator) ProbeURL(_ context.Context, value, contains string) error {
|
||||
f.publicProbes = append(f.publicProbes, value)
|
||||
if f.failPublic {
|
||||
if f.failPublic || (f.failPublicAfter > 0 && len(f.publicProbes) >= f.failPublicAfter) {
|
||||
return errors.New("injected public smoke failure")
|
||||
}
|
||||
if f.rejectMarkers && contains != "" {
|
||||
@@ -92,13 +101,44 @@ func TestPublicSmokeFailureRestoresBlueGreenHandlerAndSlot(t *testing.T) {
|
||||
if err != nil || target != old {
|
||||
t.Fatalf("green=%q err=%v", target, err)
|
||||
}
|
||||
if _, err = os.Stat(cfg.Deployment.StateFile); !os.IsNotExist(err) {
|
||||
t.Fatal("failed public smoke wrote state")
|
||||
record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if record.ActiveRelease != old || record.LastAttemptOutcome != "failed" || record.CandidateRelease != "" || record.LastAttemptRelease != fresh {
|
||||
t.Fatalf("failed attempt state=%+v", record)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContinuityFailureRestoresBlueGreenRoute(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "blue_green")
|
||||
handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
|
||||
template := filepath.Join(cfg.Deployment.Root, "handler.template")
|
||||
original := []byte("reverse_proxy 127.0.0.1:8090\n")
|
||||
_ = os.WriteFile(handler, original, 0o644)
|
||||
_ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644)
|
||||
blue := filepath.Join(cfg.Deployment.Root, "slots", "blue")
|
||||
green := filepath.Join(cfg.Deployment.Root, "slots", "green")
|
||||
_ = replaceSymlink(blue, old)
|
||||
_ = replaceSymlink(green, old)
|
||||
cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}}
|
||||
operator := &fakeOperator{active: map[string]bool{}, failPublicAfter: 3}
|
||||
if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err == nil || !strings.Contains(err.Error(), "continuity") {
|
||||
t.Fatalf("expected continuity failure, got %v", err)
|
||||
}
|
||||
body, _ := os.ReadFile(handler)
|
||||
if string(body) != string(original) {
|
||||
t.Fatalf("handler not restored: %q", body)
|
||||
}
|
||||
target, err := resolveReleaseLink(cfg.Deployment.Root, green)
|
||||
if err != nil || target != old {
|
||||
t.Fatalf("green=%q err=%v", target, err)
|
||||
}
|
||||
}
|
||||
func (f *fakeOperator) ValidateCaddy(context.Context, string) error { return nil }
|
||||
func (f *fakeOperator) ReloadCaddy(context.Context) error {
|
||||
if f.failReload {
|
||||
f.reloads++
|
||||
if f.failReload || (f.failReloadAt > 0 && f.reloads == f.failReloadAt) {
|
||||
return errors.New("injected reload failure")
|
||||
}
|
||||
return nil
|
||||
@@ -117,7 +157,7 @@ func baseConfig(t *testing.T, strategy string) (config.Config, string, string) {
|
||||
if err := os.MkdirAll(filepath.Join(root, "releases"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := filepath.Join(root, "releases", "legacy-old")
|
||||
old := filepath.Join(root, "releases", "sha256-"+strings.Repeat("c", 64))
|
||||
fresh := filepath.Join(root, "releases", "sha256-"+strings.Repeat("a", 64))
|
||||
for _, dir := range []string{old, fresh} {
|
||||
if err := os.Mkdir(dir, 0o755); err != nil {
|
||||
@@ -127,11 +167,34 @@ func baseConfig(t *testing.T, strategy string) (config.Config, string, string) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
cfg := config.Config{SchemaVersion: 2, Service: config.Service{Name: "example-site", AllowedHost: "example.test", EnvironmentFile: "/etc/tend/environment/example-site.env"}, Build: config.Build{Package: "./cmd/site", Binary: "app", Branch: "main"}, Deployment: config.Deployment{Strategy: strategy, Root: root, LockFile: filepath.Join(root, "deploy.lock"), StateFile: filepath.Join(root, "state.json"), HealthPath: "/healthz", ReadinessPath: "/readyz", CandidateTimeoutSecs: 2, Smoke: []config.Smoke{{Path: "/", Contains: "Example"}}, PublicSmoke: []config.PublicSmoke{{URL: "https://example.test/", Contains: "Example"}}}}
|
||||
cfg := config.Config{SchemaVersion: 2, Service: config.Service{Name: "example-site", AllowedHost: "example.test", EnvironmentFile: "/etc/tend/environment/example-site.env"}, Build: config.Build{Package: "./cmd/site", Binary: "app", Branch: "main"}, Deployment: config.Deployment{Strategy: strategy, Root: root, LockFile: filepath.Join(root, "deploy.lock"), StateFile: filepath.Join(root, "state.json"), EventLog: filepath.Join(root, "deployment-events.jsonl"), HealthPath: "/healthz", ReadinessPath: "/readyz", CandidateTimeoutSecs: 2, ActivationWindowSecs: 1, Smoke: []config.Smoke{{Path: "/", Contains: "Example"}}, PublicSmoke: []config.PublicSmoke{{URL: "https://example.test/", Contains: "Example"}}}}
|
||||
return cfg, old, fresh
|
||||
}
|
||||
func manager(operator Operator, fresh string) Manager {
|
||||
return Manager{Operator: operator, Now: func() time.Time { return time.Unix(100, 0).UTC() }, Prepare: func(config.Config, string, string, string) (string, error) { return fresh, nil }, Inspect: func(config.Config, string, string, string) error { return nil }}
|
||||
return Manager{Operator: operator, Now: func() time.Time { return time.Unix(100, 0).UTC() }, Prepare: func(config.Config, string, string, string) (string, error) { return fresh, nil }, Inspect: func(config.Config, string, string, string) error { return nil }, ReadIdentity: func(string) (releaseIdentity, error) {
|
||||
return releaseIdentity{Version: "v0.2.0-preview.1", Commit: strings.Repeat("b", 40)}, nil
|
||||
}, OperationID: func() (string, error) { return strings.Repeat("d", 32), nil }, Sleep: func(context.Context, time.Duration) error { return nil }}
|
||||
}
|
||||
|
||||
func singletonSettings(t *testing.T, cfg config.Config, currentRelease string) (*config.Singleton, string) {
|
||||
t.Helper()
|
||||
handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
|
||||
template := filepath.Join(cfg.Deployment.Root, "handler.template")
|
||||
if err := os.WriteFile(handler, []byte("reverse_proxy 127.0.0.1:8092\n"), 0o640); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
current := filepath.Join(cfg.Deployment.Root, "current")
|
||||
if err := replaceSymlink(current, currentRelease); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return &config.Singleton{
|
||||
Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN",
|
||||
CurrentLink: current, PreviousLink: filepath.Join(cfg.Deployment.Root, "previous"), CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"),
|
||||
CaddyHandler: handler, CaddyHandlerTemplate: template,
|
||||
}, handler
|
||||
}
|
||||
|
||||
func TestBlueGreenActivationAndRollback(t *testing.T) {
|
||||
@@ -158,13 +221,28 @@ func TestBlueGreenActivationAndRollback(t *testing.T) {
|
||||
}
|
||||
operator := &fakeOperator{active: map[string]bool{"example-blue.service": true, "example-green.service": true}}
|
||||
m := manager(operator, fresh)
|
||||
report, err := m.Deploy(context.Background(), cfg, Request{Activate: true})
|
||||
var events []eventlog.Event
|
||||
m.AppendEvent = func(_ string, event eventlog.Event) error {
|
||||
events = append(events, event)
|
||||
return nil
|
||||
}
|
||||
report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.ActiveRelease != fresh || report.PreviousRelease != old {
|
||||
t.Fatalf("report=%+v", report)
|
||||
}
|
||||
deployed, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if deployed.DesiredRelease != fresh || deployed.CandidateRelease != "" || deployed.LastAttemptRelease != fresh || deployed.LastAttemptOutcome != "succeeded" {
|
||||
t.Fatalf("deployment identity state=%+v", deployed)
|
||||
}
|
||||
if len(events) != 2 || events[0].Phase != "candidate" || events[0].Outcome != "running" || events[1].Phase != "activation" || events[1].Outcome != "succeeded" || events[0].OperationID != events[1].OperationID {
|
||||
t.Fatalf("events=%+v", events)
|
||||
}
|
||||
if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 {
|
||||
t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err)
|
||||
}
|
||||
@@ -180,6 +258,9 @@ func TestBlueGreenActivationAndRollback(t *testing.T) {
|
||||
if record.ActiveRelease != old || record.PreviousRelease != fresh {
|
||||
t.Fatalf("rollback=%+v", record)
|
||||
}
|
||||
if len(events) != 4 || events[2].Phase != "rollback" || events[2].Outcome != "running" || events[3].Phase != "rollback" || events[3].Outcome != "succeeded" || events[2].OperationID != events[3].OperationID || events[2].ArtifactDigest != strings.Repeat("c", 64) {
|
||||
t.Fatalf("rollback events=%+v", events)
|
||||
}
|
||||
if len(operator.probes) != 4 {
|
||||
t.Fatalf("rollback probes=%#v", operator.probes)
|
||||
}
|
||||
@@ -190,6 +271,58 @@ func TestBlueGreenActivationAndRollback(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeploymentEvidenceCanNeverBlockActivationOrRollback(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
|
||||
m := manager(&fakeOperator{active: map[string]bool{}}, fresh)
|
||||
appendCalls := 0
|
||||
m.AppendEvent = func(string, eventlog.Event) error {
|
||||
appendCalls++
|
||||
return errors.New("injected event failure")
|
||||
}
|
||||
report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.EventWarnings != 2 || report.ActiveRelease != fresh {
|
||||
t.Fatalf("report=%+v", report)
|
||||
}
|
||||
if _, err := m.Rollback(context.Background(), cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err != nil || record.ActiveRelease != old || appendCalls != 4 {
|
||||
t.Fatalf("record=%+v appends=%d err=%v", record, appendCalls, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeploymentEvidenceIdentityAndEntropyAreBestEffort(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
damage func(*Manager)
|
||||
}{
|
||||
{"identity", func(manager *Manager) {
|
||||
manager.ReadIdentity = func(string) (releaseIdentity, error) {
|
||||
return releaseIdentity{}, errors.New("injected identity failure")
|
||||
}
|
||||
}},
|
||||
{"entropy", func(manager *Manager) {
|
||||
manager.OperationID = func() (string, error) { return "", errors.New("injected entropy failure") }
|
||||
}},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
|
||||
manager := manager(&fakeOperator{active: map[string]bool{}}, fresh)
|
||||
test.damage(&manager)
|
||||
report, err := manager.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
|
||||
if err != nil || report.EventWarnings != 1 || report.ActiveRelease != fresh {
|
||||
t.Fatalf("report=%+v err=%v", report, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlueGreenCaddyFailureRestoresHandlerAndSlot(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "blue_green")
|
||||
handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
|
||||
@@ -215,37 +348,47 @@ func TestBlueGreenCaddyFailureRestoresHandlerAndSlot(t *testing.T) {
|
||||
if err != nil || target != old {
|
||||
t.Fatalf("green=%q err=%v", target, err)
|
||||
}
|
||||
if _, err := os.Stat(cfg.Deployment.StateFile); !os.IsNotExist(err) {
|
||||
t.Fatal("failed activation wrote state")
|
||||
failed, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if failed.ActiveRelease != old || failed.LastAttemptOutcome != "failed" || failed.CandidateRelease != "" {
|
||||
t.Fatalf("failed state=%+v", failed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingletonRestartFailureRestoresPointers(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
previous := filepath.Join(cfg.Deployment.Root, "previous")
|
||||
current := filepath.Join(cfg.Deployment.Root, "current")
|
||||
_ = replaceSymlink(current, old)
|
||||
cfg.Deployment.Singleton = &config.Singleton{Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN", CurrentLink: current, PreviousLink: previous}
|
||||
operator := &fakeOperator{active: map[string]bool{}, failRestartUnit: "example-site.service"}
|
||||
settings, handler := singletonSettings(t, cfg, old)
|
||||
cfg.Deployment.Singleton = settings
|
||||
operator := &fakeOperator{active: map[string]bool{}, failRestartUnit: "example-site.service", failRestartOnce: true}
|
||||
m := manager(operator, fresh)
|
||||
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil {
|
||||
t.Fatal("expected failure")
|
||||
}
|
||||
target, err := resolveReleaseLink(cfg.Deployment.Root, current)
|
||||
target, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink)
|
||||
if err != nil || target != old {
|
||||
t.Fatalf("current=%q err=%v", target, err)
|
||||
}
|
||||
if _, err := os.Lstat(previous); !os.IsNotExist(err) {
|
||||
if _, err := os.Lstat(cfg.Deployment.Singleton.PreviousLink); !os.IsNotExist(err) {
|
||||
t.Fatal("previous pointer was not restored")
|
||||
}
|
||||
body, err := os.ReadFile(handler)
|
||||
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
|
||||
t.Fatalf("handler=%q err=%v", body, err)
|
||||
}
|
||||
if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 {
|
||||
t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err)
|
||||
}
|
||||
if operator.active["example-site-tend-candidate.service"] {
|
||||
t.Fatal("candidate was not stopped after successful restoration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatePersistsOnlyAfterSuccessfulActivation(t *testing.T) {
|
||||
func TestStateRecordsSuccessfulActivation(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
current := filepath.Join(cfg.Deployment.Root, "current")
|
||||
previous := filepath.Join(cfg.Deployment.Root, "previous")
|
||||
_ = replaceSymlink(current, old)
|
||||
cfg.Deployment.Singleton = &config.Singleton{Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN", CurrentLink: current, PreviousLink: previous}
|
||||
settings, handler := singletonSettings(t, cfg, old)
|
||||
cfg.Deployment.Singleton = settings
|
||||
operator := &fakeOperator{active: map[string]bool{}}
|
||||
m := manager(operator, fresh)
|
||||
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err != nil {
|
||||
@@ -261,4 +404,72 @@ func TestStatePersistsOnlyAfterSuccessfulActivation(t *testing.T) {
|
||||
if operator.candidateFile != cfg.Service.EnvironmentFile || len(operator.candidateEnvironment) != 1 || operator.candidateEnvironment["EXAMPLE_LISTEN"] != "127.0.0.1:18092" {
|
||||
t.Fatalf("candidate file=%q environment=%#v", operator.candidateFile, operator.candidateEnvironment)
|
||||
}
|
||||
body, err := os.ReadFile(handler)
|
||||
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
|
||||
t.Fatalf("handler=%q err=%v", body, err)
|
||||
}
|
||||
if operator.reloads != 2 || operator.active["example-site-tend-candidate.service"] {
|
||||
t.Fatalf("reloads=%d active=%#v", operator.reloads, operator.active)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingletonContinuityFailureRestoresHandlerPointersAndService(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
settings, handler := singletonSettings(t, cfg, old)
|
||||
cfg.Deployment.Singleton = settings
|
||||
operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failPublicAfter: 4}
|
||||
m := manager(operator, fresh)
|
||||
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil || !strings.Contains(err.Error(), "continuity") {
|
||||
t.Fatalf("expected continuity failure, got %v", err)
|
||||
}
|
||||
current, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink)
|
||||
if err != nil || current != old {
|
||||
t.Fatalf("current=%q err=%v", current, err)
|
||||
}
|
||||
body, err := os.ReadFile(handler)
|
||||
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
|
||||
t.Fatalf("handler=%q err=%v", body, err)
|
||||
}
|
||||
if operator.active["example-site-tend-candidate.service"] {
|
||||
t.Fatal("candidate was not stopped after continuity restoration")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingletonCaddyReloadFailuresRestorePriorRoute(t *testing.T) {
|
||||
for _, reload := range []int{1, 2} {
|
||||
t.Run(fmt.Sprintf("reload-%d", reload), func(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
settings, handler := singletonSettings(t, cfg, old)
|
||||
cfg.Deployment.Singleton = settings
|
||||
operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failReloadAt: reload}
|
||||
if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true}); err == nil || !strings.Contains(err.Error(), "Caddy reload failed") {
|
||||
t.Fatalf("expected Caddy reload failure, got %v", err)
|
||||
}
|
||||
current, err := resolveReleaseLink(cfg.Deployment.Root, settings.CurrentLink)
|
||||
if err != nil || current != old {
|
||||
t.Fatalf("current=%q err=%v", current, err)
|
||||
}
|
||||
body, err := os.ReadFile(handler)
|
||||
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
|
||||
t.Fatalf("handler=%q err=%v", body, err)
|
||||
}
|
||||
if operator.active["example-site-tend-candidate.service"] {
|
||||
t.Fatal("candidate was not stopped after route restoration")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingletonIncompleteRecoveryKeepsProvenCandidateRunning(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
settings, _ := singletonSettings(t, cfg, old)
|
||||
cfg.Deployment.Singleton = settings
|
||||
operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failReload: true}
|
||||
_, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true})
|
||||
if err == nil || !strings.Contains(err.Error(), "candidate remains routed for operator recovery") {
|
||||
t.Fatalf("expected explicit incomplete recovery, got %v", err)
|
||||
}
|
||||
if !operator.active["example-site-tend-candidate.service"] {
|
||||
t.Fatal("proven candidate was stopped despite incomplete route restoration")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -196,6 +196,15 @@ func extractArtifact(artifact, stage string) error {
|
||||
_ = out.Close()
|
||||
return err
|
||||
}
|
||||
// OpenFile modes are filtered through the caller's umask. Tend is
|
||||
// commonly invoked by a root account with umask 0077, while release
|
||||
// binaries must remain executable by their dedicated service users.
|
||||
// Reapply the validated, name-derived mode explicitly before the file
|
||||
// becomes part of an immutable release.
|
||||
if err := out.Chmod(mode); err != nil {
|
||||
_ = out.Close()
|
||||
return err
|
||||
}
|
||||
if err := out.Sync(); err != nil {
|
||||
_ = out.Close()
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
//go:build linux
|
||||
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestExtractArtifactAppliesReleaseModesUnderRestrictiveUmask(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
artifact := filepath.Join(dir, "release.tar.gz")
|
||||
file, err := os.OpenFile(artifact, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gz := gzip.NewWriter(file)
|
||||
tw := tar.NewWriter(gz)
|
||||
entries := map[string][]byte{
|
||||
"bundle/app": []byte("executable"),
|
||||
"bundle/BUILDINFO.json": []byte("{}"),
|
||||
"bundle/RELEASE.json": []byte("{}"),
|
||||
"bundle/SBOM.spdx.json": []byte("{}"),
|
||||
"bundle/SHA256SUMS": []byte("checksums"),
|
||||
}
|
||||
for name, body := range entries {
|
||||
if err := tw.WriteHeader(&tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0o600, Size: int64(len(body))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write(body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
oldUmask := syscall.Umask(0o077)
|
||||
t.Cleanup(func() { syscall.Umask(oldUmask) })
|
||||
stage := filepath.Join(dir, "stage")
|
||||
if err := os.Mkdir(stage, 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := extractArtifact(artifact, stage); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for name, want := range map[string]os.FileMode{
|
||||
"app": 0o755,
|
||||
"BUILDINFO.json": 0o644,
|
||||
"RELEASE.json": 0o644,
|
||||
"SBOM.spdx.json": 0o644,
|
||||
"SHA256SUMS": 0o644,
|
||||
} {
|
||||
info, err := os.Stat(filepath.Join(stage, name))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := info.Mode().Perm(); got != want {
|
||||
t.Fatalf("%s mode=%#o want=%#o", name, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user