feat: publish Gamertan Tend preview source
Sanitized root snapshot from private source commit a72903c63e1753f9e6ffbf40453c0830bdfc05c5 and tree 295641e67eef5979da76746d8ae271249568263e. Private development history and workflows are excluded by the exact allowlist. AI-assisted: OpenAI Codex helped implement, test, and audit this preview. Signed-off-by: Cole Speelman <crspeelman@gmail.com>
This commit is contained in:
@@ -0,0 +1,619 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gamertan.com/tend/internal/config"
|
||||
"gamertan.com/tend/internal/state"
|
||||
)
|
||||
|
||||
type Request struct {
|
||||
Artifact string
|
||||
SHA256 string
|
||||
ApprovedSHA256 string
|
||||
Activate bool
|
||||
}
|
||||
type Report struct {
|
||||
Validated bool `json:"validated"`
|
||||
Mutation string `json:"mutation"`
|
||||
Release string `json:"release,omitempty"`
|
||||
ActiveRelease string `json:"active_release,omitempty"`
|
||||
PreviousRelease string `json:"previous_release,omitempty"`
|
||||
}
|
||||
type Status struct {
|
||||
State *state.Record `json:"state,omitempty"`
|
||||
Units map[string]bool `json:"units"`
|
||||
StateInitialized bool `json:"state_initialized"`
|
||||
}
|
||||
|
||||
type Manager struct {
|
||||
Operator Operator
|
||||
Now func() time.Time
|
||||
Prepare func(config.Config, string, string, string) (string, error)
|
||||
Inspect func(config.Config, string, string, string) error
|
||||
}
|
||||
|
||||
func NewManager(operator Operator) Manager {
|
||||
return Manager{Operator: operator, Now: time.Now, Prepare: prepareRelease, Inspect: inspectArtifact}
|
||||
}
|
||||
|
||||
func (m Manager) Deploy(ctx context.Context, cfg config.Config, request Request) (Report, error) {
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
if !request.Activate {
|
||||
if err := m.Inspect(cfg, request.Artifact, request.SHA256, request.ApprovedSHA256); err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
return Report{Validated: true, Mutation: "none"}, nil
|
||||
}
|
||||
lock, err := acquireLock(cfg.Deployment.LockFile)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
defer lock.Close()
|
||||
release, err := m.Prepare(cfg, request.Artifact, request.SHA256, request.ApprovedSHA256)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
record, err := loadOrBootstrap(cfg, m.Now())
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
switch cfg.Deployment.Strategy {
|
||||
case "blue_green":
|
||||
err = m.deployBlueGreen(ctx, cfg, record, release)
|
||||
case "singleton_candidate":
|
||||
err = m.deploySingleton(ctx, cfg, record, release)
|
||||
default:
|
||||
err = errors.New("unsupported strategy")
|
||||
}
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
updated, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err != nil {
|
||||
return Report{}, err
|
||||
}
|
||||
return Report{Validated: true, Mutation: "activated", Release: release, ActiveRelease: updated.ActiveRelease, PreviousRelease: updated.PreviousRelease}, nil
|
||||
}
|
||||
|
||||
func loadOrBootstrap(cfg config.Config, now time.Time) (state.Record, error) {
|
||||
record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err == nil {
|
||||
return record, nil
|
||||
}
|
||||
if !os.IsNotExist(err) {
|
||||
return state.Record{}, err
|
||||
}
|
||||
switch cfg.Deployment.Strategy {
|
||||
case "blue_green":
|
||||
slot := cfg.Deployment.BlueGreen.BootstrapActive
|
||||
release, err := resolveReleaseLink(cfg.Deployment.Root, slotConfig(*cfg.Deployment.BlueGreen, slot).Link)
|
||||
if err != nil {
|
||||
return state.Record{}, fmt.Errorf("bootstrap active slot: %w", err)
|
||||
}
|
||||
return state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, ActiveSlot: slot, ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil
|
||||
case "singleton_candidate":
|
||||
release, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink)
|
||||
if err != nil {
|
||||
return state.Record{}, fmt.Errorf("bootstrap singleton: %w", err)
|
||||
}
|
||||
return state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, ActiveSlot: "singleton", ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil
|
||||
}
|
||||
return state.Record{}, errors.New("unsupported strategy")
|
||||
}
|
||||
|
||||
func (m Manager) deployBlueGreen(ctx context.Context, cfg config.Config, record state.Record, release string) (err error) {
|
||||
bg := *cfg.Deployment.BlueGreen
|
||||
inactive := "blue"
|
||||
if record.ActiveSlot == "blue" {
|
||||
inactive = "green"
|
||||
}
|
||||
slot := slotConfig(bg, inactive)
|
||||
oldInactive, oldErr := resolveReleaseLink(cfg.Deployment.Root, slot.Link)
|
||||
if oldErr != nil && !os.IsNotExist(oldErr) {
|
||||
return oldErr
|
||||
}
|
||||
oldHandler, err := os.ReadFile(bg.CaddyHandler)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read current Caddy handler: %w", err)
|
||||
}
|
||||
handlerChanged := false
|
||||
linkChanged := false
|
||||
defer func() {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
if handlerChanged {
|
||||
_ = atomicWrite(bg.CaddyHandler, oldHandler, 0o644)
|
||||
_ = m.Operator.ValidateCaddy(ctx, bg.CaddyConfig)
|
||||
_ = m.Operator.ReloadCaddy(ctx)
|
||||
}
|
||||
if linkChanged {
|
||||
if oldErr == nil {
|
||||
_ = replaceSymlink(slot.Link, oldInactive)
|
||||
_ = m.Operator.Restart(ctx, slot.Unit)
|
||||
} else {
|
||||
_ = removeSymlink(slot.Link)
|
||||
_ = m.Operator.Stop(ctx, slot.Unit)
|
||||
}
|
||||
}
|
||||
}()
|
||||
if err = replaceSymlink(slot.Link, release); err != nil {
|
||||
return err
|
||||
}
|
||||
linkChanged = true
|
||||
if err = m.Operator.Restart(ctx, slot.Unit); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = m.probeAll(ctx, cfg, slot.Address); err != nil {
|
||||
return fmt.Errorf("candidate failed: %w", err)
|
||||
}
|
||||
handler, err := renderHandler(bg.CaddyHandlerTemplate, slot.Address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = atomicWrite(bg.CaddyHandler, handler, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
handlerChanged = true
|
||||
if err = m.Operator.ValidateCaddy(ctx, bg.CaddyConfig); err != nil {
|
||||
return fmt.Errorf("Caddy validation failed: %w", err)
|
||||
}
|
||||
if err = m.Operator.ReloadCaddy(ctx); err != nil {
|
||||
return fmt.Errorf("Caddy reload failed: %w", err)
|
||||
}
|
||||
if err = m.probeAll(ctx, cfg, slot.Address); err != nil {
|
||||
return fmt.Errorf("post-activation smoke failed: %w", err)
|
||||
}
|
||||
next := state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, ActiveSlot: inactive, ActiveRelease: release, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
||||
if err = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m Manager) deploySingleton(ctx context.Context, cfg config.Config, record state.Record, release string) (err error) {
|
||||
single := *cfg.Deployment.Singleton
|
||||
candidateUnit := cfg.Service.Name + "-tend-candidate.service"
|
||||
env := copyMap(single.Environment)
|
||||
env[single.ListenEnv] = single.CandidateAddress
|
||||
binary := filepath.Join(release, cfg.Build.Binary)
|
||||
if err = m.Operator.StartCandidate(ctx, candidateUnit, binary, env); err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
_ = m.Operator.Stop(stopCtx, candidateUnit)
|
||||
}()
|
||||
if err = m.probeAll(ctx, cfg, single.CandidateAddress); err != nil {
|
||||
return fmt.Errorf("candidate failed: %w", err)
|
||||
}
|
||||
oldCurrent, err := resolveReleaseLink(cfg.Deployment.Root, single.CurrentLink)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
oldPrevious, previousErr := resolveReleaseLink(cfg.Deployment.Root, single.PreviousLink)
|
||||
currentChanged := false
|
||||
previousChanged := false
|
||||
defer func() {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
if currentChanged {
|
||||
_ = replaceSymlink(single.CurrentLink, oldCurrent)
|
||||
_ = m.Operator.Restart(ctx, single.Unit)
|
||||
}
|
||||
if previousChanged {
|
||||
if previousErr == nil {
|
||||
_ = replaceSymlink(single.PreviousLink, oldPrevious)
|
||||
} else {
|
||||
_ = removeSymlink(single.PreviousLink)
|
||||
}
|
||||
}
|
||||
}()
|
||||
if err = replaceSymlink(single.PreviousLink, oldCurrent); err != nil {
|
||||
return err
|
||||
}
|
||||
previousChanged = true
|
||||
if err = replaceSymlink(single.CurrentLink, release); err != nil {
|
||||
return err
|
||||
}
|
||||
currentChanged = true
|
||||
if err = m.Operator.Restart(ctx, single.Unit); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = m.probeAll(ctx, cfg, single.Address); err != nil {
|
||||
return fmt.Errorf("post-activation smoke failed: %w", err)
|
||||
}
|
||||
next := state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, ActiveSlot: "singleton", ActiveRelease: release, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
||||
if err = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m Manager) Rollback(ctx context.Context, cfg config.Config) (state.Record, error) {
|
||||
lock, err := acquireLock(cfg.Deployment.LockFile)
|
||||
if err != nil {
|
||||
return state.Record{}, err
|
||||
}
|
||||
defer lock.Close()
|
||||
record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err != nil {
|
||||
return state.Record{}, err
|
||||
}
|
||||
if record.PreviousRelease == "" {
|
||||
return state.Record{}, errors.New("no previous release is recorded")
|
||||
}
|
||||
switch cfg.Deployment.Strategy {
|
||||
case "blue_green":
|
||||
err = m.rollbackBlueGreen(ctx, cfg, record)
|
||||
case "singleton_candidate":
|
||||
err = m.rollbackSingleton(ctx, cfg, record)
|
||||
default:
|
||||
err = errors.New("unsupported strategy")
|
||||
}
|
||||
if err != nil {
|
||||
return state.Record{}, err
|
||||
}
|
||||
return state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
}
|
||||
func (m Manager) rollbackBlueGreen(ctx context.Context, cfg config.Config, record state.Record) (err error) {
|
||||
bg := *cfg.Deployment.BlueGreen
|
||||
slot := slotConfig(bg, record.PreviousSlot)
|
||||
if active, checkErr := m.Operator.IsActive(ctx, slot.Unit); checkErr != nil {
|
||||
return checkErr
|
||||
} else if !active {
|
||||
if err = m.Operator.Restart(ctx, slot.Unit); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err = m.probeHealthReadiness(ctx, cfg, slot.Address); err != nil {
|
||||
return err
|
||||
}
|
||||
oldHandler, err := os.ReadFile(bg.CaddyHandler)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
changed := false
|
||||
defer func() {
|
||||
if err != nil && changed {
|
||||
_ = atomicWrite(bg.CaddyHandler, oldHandler, 0o644)
|
||||
_ = m.Operator.ValidateCaddy(ctx, bg.CaddyConfig)
|
||||
_ = m.Operator.ReloadCaddy(ctx)
|
||||
}
|
||||
}()
|
||||
handler, err := renderHandler(bg.CaddyHandlerTemplate, slot.Address)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = atomicWrite(bg.CaddyHandler, handler, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
changed = true
|
||||
if err = m.Operator.ValidateCaddy(ctx, bg.CaddyConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = m.Operator.ReloadCaddy(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = m.probeHealthReadiness(ctx, cfg, slot.Address); err != nil {
|
||||
return err
|
||||
}
|
||||
next := state.Record{SchemaVersion: 1, Strategy: record.Strategy, ActiveSlot: record.PreviousSlot, ActiveRelease: record.PreviousRelease, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
||||
return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next)
|
||||
}
|
||||
func (m Manager) rollbackSingleton(ctx context.Context, cfg config.Config, record state.Record) (err error) {
|
||||
single := *cfg.Deployment.Singleton
|
||||
oldCurrent, err := resolveReleaseLink(cfg.Deployment.Root, single.CurrentLink)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = replaceSymlink(single.CurrentLink, record.PreviousRelease); err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
_ = replaceSymlink(single.CurrentLink, oldCurrent)
|
||||
_ = m.Operator.Restart(ctx, single.Unit)
|
||||
}
|
||||
}()
|
||||
if err = m.Operator.Restart(ctx, single.Unit); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = m.probeHealthReadiness(ctx, cfg, single.Address); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = replaceSymlink(single.PreviousLink, record.ActiveRelease)
|
||||
next := state.Record{SchemaVersion: 1, Strategy: record.Strategy, ActiveSlot: "singleton", ActiveRelease: record.PreviousRelease, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
||||
return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next)
|
||||
}
|
||||
|
||||
func (m Manager) Status(ctx context.Context, cfg config.Config) (Status, error) {
|
||||
result := Status{Units: map[string]bool{}}
|
||||
record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err == nil {
|
||||
result.State = &record
|
||||
result.StateInitialized = true
|
||||
} else if !os.IsNotExist(err) {
|
||||
return Status{}, err
|
||||
}
|
||||
units := []string{}
|
||||
if cfg.Deployment.Strategy == "blue_green" {
|
||||
units = []string{cfg.Deployment.BlueGreen.Blue.Unit, cfg.Deployment.BlueGreen.Green.Unit}
|
||||
} else {
|
||||
units = []string{cfg.Deployment.Singleton.Unit}
|
||||
}
|
||||
for _, unit := range units {
|
||||
active, err := m.Operator.IsActive(ctx, unit)
|
||||
if err != nil {
|
||||
return Status{}, err
|
||||
}
|
||||
result.Units[unit] = active
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (m Manager) Prune(cfg config.Config, keep int, apply bool) ([]string, error) {
|
||||
if keep < 2 || keep > 100 {
|
||||
return nil, errors.New("keep must be between 2 and 100")
|
||||
}
|
||||
lock, err := acquireLock(cfg.Deployment.LockFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer lock.Close()
|
||||
record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries, err := os.ReadDir(filepath.Join(cfg.Deployment.Root, "releases"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
type candidate struct {
|
||||
name, path string
|
||||
mod time.Time
|
||||
}
|
||||
items := []candidate{}
|
||||
protected := map[string]bool{record.ActiveRelease: true, record.PreviousRelease: true}
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() || entry.Type()&os.ModeSymlink != 0 || !strings.HasPrefix(entry.Name(), "sha256-") {
|
||||
continue
|
||||
}
|
||||
path := filepath.Join(cfg.Deployment.Root, "releases", entry.Name())
|
||||
if protected[path] {
|
||||
continue
|
||||
}
|
||||
info, err := entry.Info()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, candidate{entry.Name(), path, info.ModTime()})
|
||||
}
|
||||
sort.Slice(items, func(i, j int) bool { return items[i].mod.After(items[j].mod) })
|
||||
retained := keep - 2
|
||||
if retained < 0 {
|
||||
retained = 0
|
||||
}
|
||||
if retained > len(items) {
|
||||
retained = len(items)
|
||||
}
|
||||
items = items[retained:]
|
||||
paths := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
paths = append(paths, item.path)
|
||||
if apply {
|
||||
if err := removeRelease(item.path, cfg.Deployment.Root); err != nil {
|
||||
return paths, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return paths, nil
|
||||
}
|
||||
|
||||
func (m Manager) probeAll(ctx context.Context, cfg config.Config, address string) error {
|
||||
checks := append([]config.Smoke{{Path: cfg.Deployment.HealthPath}, {Path: cfg.Deployment.ReadinessPath}}, cfg.Deployment.Smoke...)
|
||||
return m.probe(ctx, cfg, address, checks)
|
||||
}
|
||||
|
||||
func (m Manager) probeHealthReadiness(ctx context.Context, cfg config.Config, address string) error {
|
||||
checks := []config.Smoke{{Path: cfg.Deployment.HealthPath}, {Path: cfg.Deployment.ReadinessPath}}
|
||||
return m.probe(ctx, cfg, address, checks)
|
||||
}
|
||||
|
||||
func (m Manager) probe(ctx context.Context, cfg config.Config, address string, checks []config.Smoke) error {
|
||||
timeout := time.Duration(cfg.Deployment.CandidateTimeoutSecs) * time.Second
|
||||
for _, check := range checks {
|
||||
deadline := m.Now().Add(timeout)
|
||||
var last error
|
||||
for {
|
||||
attempt, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
last = m.Operator.Probe(attempt, address, cfg.Service.AllowedHost, check.Path, check.Contains)
|
||||
cancel()
|
||||
if last == nil {
|
||||
break
|
||||
}
|
||||
if !m.Now().Before(deadline) {
|
||||
return last
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func slotConfig(bg config.BlueGreen, name string) config.Slot {
|
||||
if name == "blue" {
|
||||
return bg.Blue
|
||||
}
|
||||
return bg.Green
|
||||
}
|
||||
func copyMap(source map[string]string) map[string]string {
|
||||
target := make(map[string]string, len(source)+1)
|
||||
for k, v := range source {
|
||||
target[k] = v
|
||||
}
|
||||
return target
|
||||
}
|
||||
func resolveReleaseLink(root, link string) (string, error) {
|
||||
info, err := os.Lstat(link)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink == 0 {
|
||||
return "", errors.New("release pointer is not a symlink")
|
||||
}
|
||||
target, err := os.Readlink(link)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !filepath.IsAbs(target) {
|
||||
target = filepath.Join(filepath.Dir(link), target)
|
||||
}
|
||||
target = filepath.Clean(target)
|
||||
probe := state.Record{SchemaVersion: 1, Strategy: "singleton_candidate", ActiveSlot: "singleton", ActiveRelease: target, UpdatedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)}
|
||||
if err := probe.Validate(root, "singleton_candidate"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
targetInfo, err := os.Lstat(target)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if !targetInfo.IsDir() || targetInfo.Mode()&os.ModeSymlink != 0 {
|
||||
return "", errors.New("release target must be a real directory")
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
func replaceSymlink(link, target string) error {
|
||||
if info, err := os.Lstat(link); err == nil && info.Mode()&os.ModeSymlink == 0 {
|
||||
return errors.New("refusing to replace non-symlink release pointer")
|
||||
} else if err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(link), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
stage, err := os.MkdirTemp(filepath.Dir(link), ".tend-link-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(stage)
|
||||
tmp := filepath.Join(stage, "next")
|
||||
if err := os.Symlink(target, tmp); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp, link)
|
||||
}
|
||||
func removeSymlink(path string) error {
|
||||
info, err := os.Lstat(path)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink == 0 {
|
||||
return errors.New("refusing to remove non-symlink")
|
||||
}
|
||||
return os.Remove(path)
|
||||
}
|
||||
func renderHandler(templatePath, address string) ([]byte, error) {
|
||||
b, err := os.ReadFile(templatePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
const marker = "{{UPSTREAM}}"
|
||||
if bytes := strings.Count(string(b), marker); bytes != 1 {
|
||||
return nil, errors.New("Caddy handler template must contain exactly one upstream marker")
|
||||
}
|
||||
return []byte(strings.Replace(string(b), marker, address, 1)), nil
|
||||
}
|
||||
func atomicWrite(path string, data []byte, mode os.FileMode) error {
|
||||
var existing os.FileInfo
|
||||
if info, err := os.Lstat(path); err == nil {
|
||||
if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() {
|
||||
return errors.New("refusing to replace non-regular or symlink file")
|
||||
}
|
||||
existing = info
|
||||
} else if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
identity := identityFor(existing, mode)
|
||||
dir := filepath.Dir(path)
|
||||
tmp, err := os.CreateTemp(dir, ".tend-write-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name := tmp.Name()
|
||||
ok := false
|
||||
defer func() {
|
||||
_ = tmp.Close()
|
||||
if !ok {
|
||||
_ = os.Remove(name)
|
||||
}
|
||||
}()
|
||||
if err := applyIdentity(tmp, identity); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tmp.Write(data); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tmp.Sync(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(name, path); err != nil {
|
||||
return err
|
||||
}
|
||||
ok = true
|
||||
return syncDirectory(dir)
|
||||
}
|
||||
func syncDirectory(path string) error {
|
||||
dir, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dir.Close()
|
||||
return dir.Sync()
|
||||
}
|
||||
func removeRelease(path, root string) error {
|
||||
releases := filepath.Join(root, "releases")
|
||||
rel, err := filepath.Rel(releases, path)
|
||||
if err != nil || rel == "." || rel == ".." || strings.ContainsRune(rel, filepath.Separator) || !strings.HasPrefix(rel, "sha256-") {
|
||||
return errors.New("unsafe prune target")
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return errors.New("prune target is not a real release directory")
|
||||
}
|
||||
return os.RemoveAll(path)
|
||||
}
|
||||
func Marshal(value any) ([]byte, error) {
|
||||
b, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return append(b, '\n'), nil
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gamertan.com/tend/internal/config"
|
||||
"gamertan.com/tend/internal/state"
|
||||
)
|
||||
|
||||
type fakeOperator struct {
|
||||
failReload bool
|
||||
failRestartUnit string
|
||||
rejectMarkers bool
|
||||
active map[string]bool
|
||||
starts, stops, restarts []string
|
||||
probes []string
|
||||
}
|
||||
|
||||
func (f *fakeOperator) Restart(_ context.Context, unit string) error {
|
||||
f.restarts = append(f.restarts, unit)
|
||||
if unit == f.failRestartUnit {
|
||||
return errors.New("injected restart failure")
|
||||
}
|
||||
f.active[unit] = true
|
||||
return nil
|
||||
}
|
||||
func (f *fakeOperator) Stop(_ context.Context, unit string) error {
|
||||
f.stops = append(f.stops, unit)
|
||||
f.active[unit] = false
|
||||
return nil
|
||||
}
|
||||
func (f *fakeOperator) IsActive(_ context.Context, unit string) (bool, error) {
|
||||
return f.active[unit], nil
|
||||
}
|
||||
func (f *fakeOperator) StartCandidate(_ context.Context, unit, binary string, env map[string]string) error {
|
||||
if !filepath.IsAbs(binary) || len(env) == 0 {
|
||||
return errors.New("bad candidate")
|
||||
}
|
||||
f.starts = append(f.starts, unit)
|
||||
f.active[unit] = true
|
||||
return nil
|
||||
}
|
||||
func (f *fakeOperator) ValidateCaddy(context.Context, string) error { return nil }
|
||||
func (f *fakeOperator) ReloadCaddy(context.Context) error {
|
||||
if f.failReload {
|
||||
return errors.New("injected reload failure")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func (f *fakeOperator) Probe(_ context.Context, address, host, path, contains string) error {
|
||||
f.probes = append(f.probes, address+path)
|
||||
if f.rejectMarkers && contains != "" {
|
||||
return errors.New("unexpected future-release smoke marker")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func baseConfig(t *testing.T, strategy string) (config.Config, string, string) {
|
||||
t.Helper()
|
||||
root := filepath.Join(t.TempDir(), "service")
|
||||
if err := os.MkdirAll(filepath.Join(root, "releases"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := filepath.Join(root, "releases", "legacy-old")
|
||||
fresh := filepath.Join(root, "releases", "sha256-"+strings.Repeat("a", 64))
|
||||
for _, dir := range []string{old, fresh} {
|
||||
if err := os.Mkdir(dir, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "app"), []byte("x"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
cfg := config.Config{SchemaVersion: 1, Service: config.Service{Name: "example-site", AllowedHost: "example.test"}, Build: config.Build{Package: "./cmd/site", Binary: "app", Branch: "main"}, Deployment: config.Deployment{Strategy: strategy, Root: root, LockFile: filepath.Join(root, "deploy.lock"), StateFile: filepath.Join(root, "state.json"), HealthPath: "/healthz", ReadinessPath: "/readyz", CandidateTimeoutSecs: 2, Smoke: []config.Smoke{{Path: "/", Contains: "Example"}}}}
|
||||
return cfg, old, fresh
|
||||
}
|
||||
func manager(operator Operator, fresh string) Manager {
|
||||
return Manager{Operator: operator, Now: func() time.Time { return time.Unix(100, 0).UTC() }, Prepare: func(config.Config, string, string, string) (string, error) { return fresh, nil }, Inspect: func(config.Config, string, string, string) error { return nil }}
|
||||
}
|
||||
|
||||
func TestBlueGreenActivationAndRollback(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "blue_green")
|
||||
handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
|
||||
template := filepath.Join(cfg.Deployment.Root, "handler.template")
|
||||
if err := os.WriteFile(handler, []byte("reverse_proxy 127.0.0.1:8090\n"), 0o640); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
blue := filepath.Join(cfg.Deployment.Root, "slots", "blue")
|
||||
green := filepath.Join(cfg.Deployment.Root, "slots", "green")
|
||||
if err := replaceSymlink(blue, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := replaceSymlink(green, old); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}}
|
||||
if err := cfg.Validate(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
operator := &fakeOperator{active: map[string]bool{"example-blue.service": true, "example-green.service": true}}
|
||||
m := manager(operator, fresh)
|
||||
report, err := m.Deploy(context.Background(), cfg, Request{Activate: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if report.ActiveRelease != fresh || report.PreviousRelease != old {
|
||||
t.Fatalf("report=%+v", report)
|
||||
}
|
||||
if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 {
|
||||
t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err)
|
||||
}
|
||||
if target, err := resolveReleaseLink(cfg.Deployment.Root, green); err != nil || target != fresh {
|
||||
t.Fatalf("green=%q err=%v", target, err)
|
||||
}
|
||||
operator.rejectMarkers = true
|
||||
operator.probes = nil
|
||||
record, err := m.Rollback(context.Background(), cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if record.ActiveRelease != old || record.PreviousRelease != fresh {
|
||||
t.Fatalf("rollback=%+v", record)
|
||||
}
|
||||
if len(operator.probes) != 4 {
|
||||
t.Fatalf("rollback probes=%#v", operator.probes)
|
||||
}
|
||||
for _, probe := range operator.probes {
|
||||
if !strings.HasSuffix(probe, "/healthz") && !strings.HasSuffix(probe, "/readyz") {
|
||||
t.Fatalf("rollback applied future-release smoke checks: %#v", operator.probes)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBlueGreenCaddyFailureRestoresHandlerAndSlot(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "blue_green")
|
||||
handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
|
||||
template := filepath.Join(cfg.Deployment.Root, "handler.template")
|
||||
original := []byte("reverse_proxy 127.0.0.1:8090\n")
|
||||
_ = os.WriteFile(handler, original, 0o644)
|
||||
_ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644)
|
||||
blue := filepath.Join(cfg.Deployment.Root, "slots", "blue")
|
||||
green := filepath.Join(cfg.Deployment.Root, "slots", "green")
|
||||
_ = replaceSymlink(blue, old)
|
||||
_ = replaceSymlink(green, old)
|
||||
cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}}
|
||||
operator := &fakeOperator{active: map[string]bool{}, failReload: true}
|
||||
m := manager(operator, fresh)
|
||||
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil {
|
||||
t.Fatal("expected failure")
|
||||
}
|
||||
body, _ := os.ReadFile(handler)
|
||||
if string(body) != string(original) {
|
||||
t.Fatalf("handler not restored: %q", body)
|
||||
}
|
||||
target, err := resolveReleaseLink(cfg.Deployment.Root, green)
|
||||
if err != nil || target != old {
|
||||
t.Fatalf("green=%q err=%v", target, err)
|
||||
}
|
||||
if _, err := os.Stat(cfg.Deployment.StateFile); !os.IsNotExist(err) {
|
||||
t.Fatal("failed activation wrote state")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSingletonRestartFailureRestoresPointers(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
previous := filepath.Join(cfg.Deployment.Root, "previous")
|
||||
current := filepath.Join(cfg.Deployment.Root, "current")
|
||||
_ = replaceSymlink(current, old)
|
||||
cfg.Deployment.Singleton = &config.Singleton{Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN", CurrentLink: current, PreviousLink: previous}
|
||||
operator := &fakeOperator{active: map[string]bool{}, failRestartUnit: "example-site.service"}
|
||||
m := manager(operator, fresh)
|
||||
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil {
|
||||
t.Fatal("expected failure")
|
||||
}
|
||||
target, err := resolveReleaseLink(cfg.Deployment.Root, current)
|
||||
if err != nil || target != old {
|
||||
t.Fatalf("current=%q err=%v", target, err)
|
||||
}
|
||||
if _, err := os.Lstat(previous); !os.IsNotExist(err) {
|
||||
t.Fatal("previous pointer was not restored")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatePersistsOnlyAfterSuccessfulActivation(t *testing.T) {
|
||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||
current := filepath.Join(cfg.Deployment.Root, "current")
|
||||
previous := filepath.Join(cfg.Deployment.Root, "previous")
|
||||
_ = replaceSymlink(current, old)
|
||||
cfg.Deployment.Singleton = &config.Singleton{Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN", CurrentLink: current, PreviousLink: previous}
|
||||
operator := &fakeOperator{active: map[string]bool{}}
|
||||
m := manager(operator, fresh)
|
||||
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if record.ActiveRelease != fresh || record.PreviousRelease != old {
|
||||
t.Fatalf("state=%+v", record)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
//go:build linux
|
||||
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
type fileLock struct{ file *os.File }
|
||||
|
||||
func acquireLock(path string) (*fileLock, error) {
|
||||
file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
|
||||
_ = file.Close()
|
||||
return nil, errors.New("deployment lock is held")
|
||||
}
|
||||
return &fileLock{file: file}, nil
|
||||
}
|
||||
func (l *fileLock) Close() error {
|
||||
if l == nil || l.file == nil {
|
||||
return nil
|
||||
}
|
||||
_ = syscall.Flock(int(l.file.Fd()), syscall.LOCK_UN)
|
||||
return l.file.Close()
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
//go:build !linux
|
||||
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package deploy
|
||||
|
||||
import "errors"
|
||||
|
||||
type fileLock struct{}
|
||||
|
||||
func acquireLock(string) (*fileLock, error) {
|
||||
return nil, errors.New("deployment mutations require Linux")
|
||||
}
|
||||
func (*fileLock) Close() error { return nil }
|
||||
@@ -0,0 +1,111 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gamertan.com/tend/internal/process"
|
||||
)
|
||||
|
||||
type Operator interface {
|
||||
Restart(context.Context, string) error
|
||||
Stop(context.Context, string) error
|
||||
IsActive(context.Context, string) (bool, error)
|
||||
StartCandidate(context.Context, string, string, map[string]string) error
|
||||
ValidateCaddy(context.Context, string) error
|
||||
ReloadCaddy(context.Context) error
|
||||
Probe(context.Context, string, string, string, string) error
|
||||
}
|
||||
|
||||
type SystemOperator struct {
|
||||
Runner process.Runner
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func (o SystemOperator) Restart(ctx context.Context, unit string) error {
|
||||
_, err := o.Runner.Run(ctx, "/", nil, "systemctl", "restart", unit)
|
||||
return err
|
||||
}
|
||||
func (o SystemOperator) Stop(ctx context.Context, unit string) error {
|
||||
_, err := o.Runner.Run(ctx, "/", nil, "systemctl", "stop", unit)
|
||||
return err
|
||||
}
|
||||
func (o SystemOperator) IsActive(ctx context.Context, unit string) (bool, error) {
|
||||
out, err := o.Runner.Run(ctx, "/", nil, "systemctl", "is-active", unit)
|
||||
if err != nil {
|
||||
if strings.TrimSpace(string(out)) == "inactive" || strings.TrimSpace(string(out)) == "failed" {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return strings.TrimSpace(string(out)) == "active", nil
|
||||
}
|
||||
func (o SystemOperator) StartCandidate(ctx context.Context, unit, binary string, env map[string]string) error {
|
||||
args := []string{
|
||||
"--unit", unit, "--collect",
|
||||
"--property=DynamicUser=yes", "--property=NoNewPrivileges=yes",
|
||||
"--property=PrivateDevices=yes", "--property=PrivateTmp=yes",
|
||||
"--property=ProtectClock=yes", "--property=ProtectControlGroups=yes",
|
||||
"--property=ProtectHome=yes", "--property=ProtectHostname=yes",
|
||||
"--property=ProtectKernelLogs=yes", "--property=ProtectKernelModules=yes",
|
||||
"--property=ProtectKernelTunables=yes", "--property=ProtectSystem=strict",
|
||||
"--property=RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX",
|
||||
"--property=RestrictNamespaces=yes", "--property=RestrictRealtime=yes",
|
||||
"--property=RestrictSUIDSGID=yes", "--property=LockPersonality=yes",
|
||||
"--property=MemoryDenyWriteExecute=yes", "--property=CapabilityBoundingSet=",
|
||||
"--property=AmbientCapabilities=",
|
||||
}
|
||||
keys := make([]string, 0, len(env))
|
||||
for key := range env {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
args = append(args, "--setenv", key+"="+env[key])
|
||||
}
|
||||
args = append(args, "--", binary)
|
||||
_, err := o.Runner.Run(ctx, "/", nil, "systemd-run", args...)
|
||||
return err
|
||||
}
|
||||
func (o SystemOperator) ValidateCaddy(ctx context.Context, path string) error {
|
||||
_, err := o.Runner.Run(ctx, "/", nil, "caddy", "validate", "--config", path, "--adapter", "caddyfile")
|
||||
return err
|
||||
}
|
||||
func (o SystemOperator) ReloadCaddy(ctx context.Context) error {
|
||||
_, err := o.Runner.Run(ctx, "/", nil, "systemctl", "reload", "caddy.service")
|
||||
return err
|
||||
}
|
||||
func (o SystemOperator) Probe(ctx context.Context, address, host, path, contains string) error {
|
||||
u := url.URL{Scheme: "http", Host: address, Path: path}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Host = host
|
||||
client := &http.Client{Timeout: o.Timeout, CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("redirect refused") }}
|
||||
response, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("probe returned HTTP %d", response.StatusCode)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, 1<<20))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if contains != "" && !strings.Contains(string(body), contains) {
|
||||
return errors.New("probe response omitted required marker")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type recordingRunner struct {
|
||||
name string
|
||||
args []string
|
||||
}
|
||||
|
||||
func (r *recordingRunner) Run(_ context.Context, _ string, _ map[string]string, name string, args ...string) ([]byte, error) {
|
||||
r.name = name
|
||||
r.args = append([]string(nil), args...)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestStartCandidateUsesArgumentVectorAndHardenedUnit(t *testing.T) {
|
||||
runner := &recordingRunner{}
|
||||
operator := SystemOperator{Runner: runner}
|
||||
env := map[string]string{"Z_ENV": "safe value", "A_ENV": "first"}
|
||||
if err := operator.StartCandidate(context.Background(), "example-tend-candidate.service", "/opt/example/releases/sha256-a/app", env); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if runner.name != "systemd-run" {
|
||||
t.Fatalf("command=%q", runner.name)
|
||||
}
|
||||
required := []string{"--property=DynamicUser=yes", "--property=NoNewPrivileges=yes", "--property=ProtectSystem=strict", "--property=MemoryDenyWriteExecute=yes", "--property=CapabilityBoundingSet=", "--setenv", "A_ENV=first", "--setenv", "Z_ENV=safe value", "--", "/opt/example/releases/sha256-a/app"}
|
||||
cursor := 0
|
||||
for _, arg := range runner.args {
|
||||
if cursor < len(required) && arg == required[cursor] {
|
||||
cursor++
|
||||
}
|
||||
}
|
||||
if cursor != len(required) {
|
||||
t.Fatalf("arguments omitted ordered security boundary: %#v", runner.args)
|
||||
}
|
||||
if reflect.DeepEqual(runner.args, []string{"sh", "-c"}) {
|
||||
t.Fatal("candidate command used a shell")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
//go:build linux
|
||||
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
type fileIdentity struct {
|
||||
mode os.FileMode
|
||||
uid, gid int
|
||||
owned bool
|
||||
}
|
||||
|
||||
func identityFor(info os.FileInfo, fallback os.FileMode) fileIdentity {
|
||||
identity := fileIdentity{mode: fallback}
|
||||
if info == nil {
|
||||
return identity
|
||||
}
|
||||
identity.mode = info.Mode().Perm()
|
||||
if stat, ok := info.Sys().(*syscall.Stat_t); ok {
|
||||
identity.uid = int(stat.Uid)
|
||||
identity.gid = int(stat.Gid)
|
||||
identity.owned = true
|
||||
}
|
||||
return identity
|
||||
}
|
||||
func applyIdentity(file *os.File, identity fileIdentity) error {
|
||||
if identity.owned {
|
||||
if err := file.Chown(identity.uid, identity.gid); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return file.Chmod(identity.mode)
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
//go:build !linux
|
||||
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package deploy
|
||||
|
||||
import "os"
|
||||
|
||||
type fileIdentity struct{ mode os.FileMode }
|
||||
|
||||
func identityFor(info os.FileInfo, fallback os.FileMode) fileIdentity {
|
||||
if info != nil {
|
||||
return fileIdentity{mode: info.Mode().Perm()}
|
||||
}
|
||||
return fileIdentity{mode: fallback}
|
||||
}
|
||||
func applyIdentity(file *os.File, identity fileIdentity) error { return file.Chmod(identity.mode) }
|
||||
@@ -0,0 +1,309 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gamertan.com/tend/internal/config"
|
||||
"gamertan.com/tend/internal/packager"
|
||||
)
|
||||
|
||||
const maxArtifactSize int64 = 512 << 20
|
||||
|
||||
func prepareRelease(cfg config.Config, artifact, expected, approved string) (string, error) {
|
||||
if err := checkArtifact(artifact, expected, approved); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := ensureTree(cfg.Deployment.Root); err != nil {
|
||||
return "", err
|
||||
}
|
||||
releases := filepath.Join(cfg.Deployment.Root, "releases")
|
||||
release := filepath.Join(releases, "sha256-"+expected)
|
||||
if info, err := os.Lstat(release); err == nil {
|
||||
if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return "", errors.New("existing release is not a directory")
|
||||
}
|
||||
if err := validateRelease(cfg, release); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return release, nil
|
||||
} else if !os.IsNotExist(err) {
|
||||
return "", err
|
||||
}
|
||||
stage := filepath.Join(releases, ".tend-stage-"+expected)
|
||||
if err := os.Mkdir(stage, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
ok := false
|
||||
defer func() {
|
||||
if !ok {
|
||||
_ = os.RemoveAll(stage)
|
||||
}
|
||||
}()
|
||||
if err := extractArtifact(artifact, stage); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := validateRelease(cfg, stage); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Chmod(stage, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := os.Rename(stage, release); err != nil {
|
||||
return "", err
|
||||
}
|
||||
ok = true
|
||||
return release, nil
|
||||
}
|
||||
|
||||
func inspectArtifact(cfg config.Config, artifact, expected, approved string) error {
|
||||
if err := checkArtifact(artifact, expected, approved); err != nil {
|
||||
return err
|
||||
}
|
||||
stage, err := os.MkdirTemp("", "tend-inspect-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.RemoveAll(stage)
|
||||
if err := extractArtifact(artifact, stage); err != nil {
|
||||
return err
|
||||
}
|
||||
return validateRelease(cfg, stage)
|
||||
}
|
||||
func checkArtifact(artifact, expected, approved string) error {
|
||||
if expected != approved || len(expected) != 64 {
|
||||
return errors.New("artifact digest was not explicitly approved")
|
||||
}
|
||||
if _, err := hex.DecodeString(expected); err != nil {
|
||||
return errors.New("artifact digest is not hexadecimal")
|
||||
}
|
||||
if !filepath.IsAbs(artifact) || filepath.Clean(artifact) != artifact {
|
||||
return errors.New("artifact path must be a clean absolute path")
|
||||
}
|
||||
info, err := os.Lstat(artifact)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Size() > maxArtifactSize {
|
||||
return errors.New("artifact must be a bounded regular file")
|
||||
}
|
||||
actual, err := fileSHA(artifact)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if actual != expected {
|
||||
return errors.New("artifact digest does not match")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureTree(root string) error {
|
||||
if os.Geteuid() != 0 && strings.HasPrefix(root, "/opt/") {
|
||||
return errors.New("deployment under /opt requires root")
|
||||
}
|
||||
if err := rejectSymlinkAncestors(root); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(root, "releases"), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
return rejectSymlinkAncestors(filepath.Join(root, "releases"))
|
||||
}
|
||||
func rejectSymlinkAncestors(path string) error {
|
||||
clean := filepath.Clean(path)
|
||||
parts := strings.Split(strings.TrimPrefix(clean, string(filepath.Separator)), string(filepath.Separator))
|
||||
current := string(filepath.Separator)
|
||||
for _, part := range parts {
|
||||
current = filepath.Join(current, part)
|
||||
info, err := os.Lstat(current)
|
||||
if os.IsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("symlink ancestor refused: %s", current)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return fmt.Errorf("non-directory ancestor refused: %s", current)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func extractArtifact(artifact, stage string) error {
|
||||
file, err := os.Open(artifact)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
gz, err := gzip.NewReader(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer gz.Close()
|
||||
tr := tar.NewReader(io.LimitReader(gz, maxArtifactSize))
|
||||
files := 0
|
||||
var total int64
|
||||
for {
|
||||
header, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
clean := filepath.Clean(filepath.FromSlash(header.Name))
|
||||
parts := strings.Split(clean, string(filepath.Separator))
|
||||
if len(parts) == 1 && header.Typeflag == tar.TypeDir {
|
||||
continue
|
||||
}
|
||||
if len(parts) != 2 || parts[0] != "bundle" || parts[1] == "" || parts[1] == "." || parts[1] == ".." {
|
||||
return fmt.Errorf("unsafe archive path %q", header.Name)
|
||||
}
|
||||
if header.Typeflag != tar.TypeReg || header.Size < 0 {
|
||||
return errors.New("archive may contain only regular files")
|
||||
}
|
||||
files++
|
||||
total += header.Size
|
||||
if files > 16 || total > maxArtifactSize {
|
||||
return errors.New("artifact exceeds extraction bounds")
|
||||
}
|
||||
target := filepath.Join(stage, parts[1])
|
||||
mode := os.FileMode(0o644)
|
||||
if !strings.HasSuffix(parts[1], ".json") && parts[1] != "SHA256SUMS" {
|
||||
mode = 0o755
|
||||
}
|
||||
out, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, mode)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := io.CopyN(out, tr, header.Size); err != nil {
|
||||
_ = out.Close()
|
||||
return err
|
||||
}
|
||||
if err := out.Sync(); err != nil {
|
||||
_ = out.Close()
|
||||
return err
|
||||
}
|
||||
if err := out.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if files < 5 {
|
||||
return errors.New("artifact is incomplete")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateRelease(cfg config.Config, release string) error {
|
||||
manifestPath := filepath.Join(release, "RELEASE.json")
|
||||
b, err := os.ReadFile(manifestPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dec := json.NewDecoder(strings.NewReader(string(b)))
|
||||
dec.DisallowUnknownFields()
|
||||
var manifest packager.Manifest
|
||||
if err := dec.Decode(&manifest); err != nil {
|
||||
return err
|
||||
}
|
||||
var trailing any
|
||||
if err := dec.Decode(&trailing); !errors.Is(err, io.EOF) {
|
||||
return errors.New("release manifest contains trailing data")
|
||||
}
|
||||
if manifest.SchemaVersion != 1 || manifest.Service != cfg.Service.Name || manifest.Binary != cfg.Build.Binary || manifest.GOOS != "linux" || manifest.GOARCH != "amd64" || manifest.CGOEnabled {
|
||||
return errors.New("release manifest does not match configuration")
|
||||
}
|
||||
if matched, _ := regexp.MatchString(`^[0-9a-f]{40}$`, manifest.Commit); !matched {
|
||||
return errors.New("release manifest commit is invalid")
|
||||
}
|
||||
binary := filepath.Join(release, cfg.Build.Binary)
|
||||
sum, err := fileSHA(binary)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if sum != manifest.BinarySHA256 {
|
||||
return errors.New("release binary digest does not match manifest")
|
||||
}
|
||||
if err := verifySums(release); err != nil {
|
||||
return err
|
||||
}
|
||||
entries, err := os.ReadDir(release)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
allowed := map[string]bool{cfg.Build.Binary: true, "BUILDINFO.json": true, "RELEASE.json": true, "SBOM.spdx.json": true, "SHA256SUMS": true}
|
||||
if len(entries) != len(allowed) {
|
||||
return errors.New("release contains unexpected files")
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if !allowed[entry.Name()] || !entry.Type().IsRegular() {
|
||||
return fmt.Errorf("unexpected release entry %s", entry.Name())
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func verifySums(release string) error {
|
||||
b, err := os.ReadFile(filepath.Join(release, "SHA256SUMS"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lines := strings.Split(strings.TrimSpace(string(b)), "\n")
|
||||
if len(lines) != 4 {
|
||||
return errors.New("SHA256SUMS must cover four release files")
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, line := range lines {
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != 2 || len(fields[0]) != 64 {
|
||||
return errors.New("malformed SHA256SUMS")
|
||||
}
|
||||
name := fields[1]
|
||||
if filepath.Base(name) != name || seen[name] {
|
||||
return errors.New("unsafe or duplicate checksum entry")
|
||||
}
|
||||
seen[name] = true
|
||||
actual, err := fileSHA(filepath.Join(release, name))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if actual != fields[0] {
|
||||
return fmt.Errorf("checksum mismatch for %s", name)
|
||||
}
|
||||
}
|
||||
required := []string{"BUILDINFO.json", "RELEASE.json", "SBOM.spdx.json"}
|
||||
sort.Strings(required)
|
||||
for _, name := range required {
|
||||
if !seen[name] {
|
||||
return fmt.Errorf("checksum omitted %s", name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
func fileSHA(path string) (string, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
// SPDX-License-Identifier: AGPL-3.0-only
|
||||
|
||||
package deploy
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func writeHostileArchive(t *testing.T, name string, typeflag byte) {
|
||||
t.Helper()
|
||||
path := filepath.Join(t.TempDir(), "bad.tar.gz")
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gz := gzip.NewWriter(file)
|
||||
tw := tar.NewWriter(gz)
|
||||
body := []byte("x")
|
||||
if err := tw.WriteHeader(&tar.Header{Name: name, Typeflag: typeflag, Mode: 0o644, Size: int64(len(body))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if typeflag == tar.TypeReg {
|
||||
_, _ = tw.Write(body)
|
||||
}
|
||||
_ = tw.Close()
|
||||
_ = gz.Close()
|
||||
_ = file.Close()
|
||||
stage := filepath.Join(t.TempDir(), "stage")
|
||||
_ = os.Mkdir(stage, 0o700)
|
||||
if err := extractArtifact(path, stage); err == nil {
|
||||
t.Fatalf("accepted hostile entry %q type %d", name, typeflag)
|
||||
}
|
||||
}
|
||||
func TestExtractionRejectsTraversalAndLinks(t *testing.T) {
|
||||
writeHostileArchive(t, "bundle/../../escape", tar.TypeReg)
|
||||
writeHostileArchive(t, "bundle/link", tar.TypeSymlink)
|
||||
}
|
||||
Reference in New Issue
Block a user