Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
10cf85c78d | ||
|
|
bf56dbce0f |
@@ -1 +1 @@
|
|||||||
{"schema_version":1,"source_commit":"8aab3db43f35e6a49aa497f45d73701b13fc9f32","source_tree":"992132ea4703437dc13ffdbb04a077816c02caf9","source_date_epoch":1787049308,"file_count":76}
|
{"schema_version":1,"source_commit":"07c1655921f21ee5e4fc4d85639d199e8867b17d","source_tree":"0a27c7c0d2fb8954e326044d06826cf7e1a37818","source_date_epoch":1787103648,"file_count":81}
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
9c6153ab5b861ec5818591269b2a38a14239f68e89911655aaebec444f54a26d PUBLIC-SNAPSHOT.json
|
a38ba7894dede461d7aec3f1b07c5c6f65728f2971b973fc81c469c32667d38e PUBLIC-SNAPSHOT.json
|
||||||
|
|||||||
@@ -1,5 +1,17 @@
|
|||||||
# Gamertan Tend
|
# Gamertan Tend
|
||||||
|
|
||||||
|
## Final public CLI preview
|
||||||
|
|
||||||
|
`v0.2.0-preview.2` is the final public Tend CLI preview. This repository and
|
||||||
|
its published AGPL releases remain available as an archived record, but it is
|
||||||
|
not accepting new feature development. Private research continues around a
|
||||||
|
networked management experience; that work is not mirrored or published here
|
||||||
|
and carries no public release commitment.
|
||||||
|
|
||||||
|
Existing licences and release artifacts are unchanged. Do not interpret this
|
||||||
|
notice as withdrawing rights already granted by the AGPL-3.0-only and 0BSD
|
||||||
|
files in this repository.
|
||||||
|
|
||||||
Tend is an opinionated release and deployment tool for small Go services on
|
Tend is an opinionated release and deployment tool for small Go services on
|
||||||
Linux, systemd, and Caddy. It packages a clean pushed commit, records exact
|
Linux, systemd, and Caddy. It packages a clean pushed commit, records exact
|
||||||
build provenance, activates a health-checked candidate, and keeps rollback
|
build provenance, activates a health-checked candidate, and keeps rollback
|
||||||
@@ -18,15 +30,24 @@ shell hooks. Application-specific data activation remains application-specific.
|
|||||||
```text
|
```text
|
||||||
tend check --config tend.json
|
tend check --config tend.json
|
||||||
tend package --config tend.json --version v0.2.0-preview.2 --out dist
|
tend package --config tend.json --version v0.2.0-preview.2 --out dist
|
||||||
|
tend inspect --config tend.json --artifact FILE --sha256 HEX --approve-sha256 HEX
|
||||||
tend push --target tend-deploy@host --known-hosts FILE --service NAME --artifact FILE --sha256 HEX --approve-sha256 HEX
|
tend push --target tend-deploy@host --known-hosts FILE --service NAME --artifact FILE --sha256 HEX --approve-sha256 HEX
|
||||||
tend receive --policy /etc/tend/receive-policy.json
|
tend receive --policy /etc/tend/receive-policy.json
|
||||||
tend check-server --policy /etc/tend/receive-policy.json
|
tend check-server --policy /etc/tend/receive-policy.json
|
||||||
tend deploy --config /etc/tend/services/example-site.json --artifact FILE --sha256 HEX --approve-sha256 HEX
|
tend deploy --config /etc/tend/services/example-site.json --artifact FILE --sha256 HEX --approve-sha256 HEX
|
||||||
tend status --config /etc/tend/services/example-site.json
|
tend status --config /etc/tend/services/example-site.json
|
||||||
|
tend reconcile --config /etc/tend/services/example-site.json --json
|
||||||
tend rollback --config /etc/tend/services/example-site.json
|
tend rollback --config /etc/tend/services/example-site.json
|
||||||
tend prune --config /etc/tend/services/example-site.json --keep 3 [--apply]
|
tend prune --config /etc/tend/services/example-site.json --keep 3 [--apply]
|
||||||
```
|
```
|
||||||
|
|
||||||
|
`inspect` performs the complete archive, checksum, approval, manifest, SBOM,
|
||||||
|
and binary-identity validation without staging or activating a release.
|
||||||
|
`reconcile` is also read-only: it compares Tend's journal with conventional
|
||||||
|
release symlinks, systemd units, the installed release identity, and the Caddy
|
||||||
|
handler. It reports drift and retained-candidate residue without silently
|
||||||
|
"repairing" a service.
|
||||||
|
|
||||||
`push` transfers one approved artifact through a pinned OpenSSH connection. A
|
`push` transfers one approved artifact through a pinned OpenSSH connection. A
|
||||||
forced, no-shell receiver maps the service name to one root-owned configuration;
|
forced, no-shell receiver maps the service name to one root-owned configuration;
|
||||||
it accepts no remote path, environment value, URL, or shell fragment. Production
|
it accepts no remote path, environment value, URL, or shell fragment. Production
|
||||||
@@ -47,6 +68,17 @@ origins for the activation window. Blue/green deployments simultaneously keep
|
|||||||
checking the previous slot, restoring the prior handler and inactive-slot state
|
checking the previous slot, restoring the prior handler and inactive-slot state
|
||||||
if routed traffic or continuity fails.
|
if routed traffic or continuity fails.
|
||||||
|
|
||||||
|
Singleton activation and rollback candidates use an operation-scoped systemd
|
||||||
|
unit and persist a bounded lease containing the operation, release, unit,
|
||||||
|
address, and start time in a separate adjacent file. Conventional deployment
|
||||||
|
state remains schema 1 so a retained older binary can still read active and
|
||||||
|
previous release identities; operators must reconcile before downgrading while
|
||||||
|
a lease exists. `tend
|
||||||
|
reconcile --json` compares that lease with unit activity, release pointers, and
|
||||||
|
the exact Caddy handler file without changing any of them. It deliberately does
|
||||||
|
not stop or clear a retained candidate: that process may still be the only
|
||||||
|
healthy route after an interrupted recovery.
|
||||||
|
|
||||||
Dry-run and digest approval are intentional friction. See the
|
Dry-run and digest approval are intentional friction. See the
|
||||||
[schema-2 migration guide](docs/SCHEMA_V2_MIGRATION.md) and the
|
[schema-2 migration guide](docs/SCHEMA_V2_MIGRATION.md) and the
|
||||||
[two-service walkthrough](docs/WALKTHROUGH.md).
|
[two-service walkthrough](docs/WALKTHROUGH.md).
|
||||||
@@ -62,12 +94,10 @@ maintenance releases and explicit rollback/reactivation for both Gamertan and
|
|||||||
the Sandwich Hime website using one reviewed candidate. See the dated
|
the Sandwich Hime website using one reviewed candidate. See the dated
|
||||||
[dogfood evidence](docs/DOGFOOD_EVIDENCE.md) for exact scope and limitations.
|
[dogfood evidence](docs/DOGFOOD_EVIDENCE.md) for exact scope and limitations.
|
||||||
|
|
||||||
The additive `v0.2.0-preview.2` candidate keeps that transport and activation
|
The additive `v0.2.0-preview.2` release keeps that transport and activation
|
||||||
contract while adding bounded deployment-event JSONL, routed-origin continuity,
|
contract while adding bounded deployment-event JSONL, routed-origin continuity,
|
||||||
rollback annotations, and the operational findings recorded through real
|
rollback annotations, and the operational findings recorded through real
|
||||||
dogfooding. Preview 1 remains unchanged. Preview 2 will not be tagged until one
|
dogfooding. Preview 1 remains unchanged.
|
||||||
identical binary has deployed, rolled back, and reactivated Gamertan, the
|
|
||||||
Sandwich Hime website, and Gamertan Observatory.
|
|
||||||
Operational friction discovered while applying the same contract to new
|
Operational friction discovered while applying the same contract to new
|
||||||
services is tracked separately in the
|
services is tracked separately in the
|
||||||
[dogfood friction ledger](docs/DOGFOOD_FRICTION.md). The ledger preserves the
|
[dogfood friction ledger](docs/DOGFOOD_FRICTION.md). The ledger preserves the
|
||||||
|
|||||||
@@ -32,6 +32,13 @@ Gamertan, the Sandwich Hime website, and Gamertan Observatory before the tag is
|
|||||||
created. EQL is not part of this generic gate; its SQLite/catalog publication
|
created. EQL is not part of this generic gate; its SQLite/catalog publication
|
||||||
needs a dedicated adapter rather than arbitrary hooks.
|
needs a dedicated adapter rather than arbitrary hooks.
|
||||||
|
|
||||||
|
The operation-scoped lease compatibility candidate at private commit
|
||||||
|
`789976ce766fd457ca10762540135e4e0e74cfe3` has completed the Gamertan and
|
||||||
|
Sandwich Hime deployment, rollback, reactivation, and failure gates. It has not
|
||||||
|
completed the Observatory gate because the live service now uses Docker
|
||||||
|
Compose, outside the released strategy contract. Do not create the Preview 2
|
||||||
|
tag from this candidate until that explicit topology decision is resolved.
|
||||||
|
|
||||||
`v0.1.0-preview.1` is immutable but withdrawn: its source and module checksums
|
`v0.1.0-preview.1` is immutable but withdrawn: its source and module checksums
|
||||||
are valid, while a fresh `go install` reports the development identity because
|
are valid, while a fresh `go install` reports the development identity because
|
||||||
the CLI did not yet adopt the tagged module version from Go build information.
|
the CLI did not yet adopt the tagged module version from Go build information.
|
||||||
|
|||||||
+59
-1
@@ -43,6 +43,8 @@ func run() error {
|
|||||||
return packageCommand(ctx, os.Args[2:])
|
return packageCommand(ctx, os.Args[2:])
|
||||||
case "deploy":
|
case "deploy":
|
||||||
return deployCommand(ctx, os.Args[2:])
|
return deployCommand(ctx, os.Args[2:])
|
||||||
|
case "inspect":
|
||||||
|
return inspectCommand(ctx, os.Args[2:])
|
||||||
case "push":
|
case "push":
|
||||||
return pushCommand(ctx, os.Args[2:])
|
return pushCommand(ctx, os.Args[2:])
|
||||||
case "receive":
|
case "receive":
|
||||||
@@ -51,6 +53,8 @@ func run() error {
|
|||||||
return checkServerCommand(os.Args[2:])
|
return checkServerCommand(os.Args[2:])
|
||||||
case "status":
|
case "status":
|
||||||
return statusCommand(ctx, os.Args[2:])
|
return statusCommand(ctx, os.Args[2:])
|
||||||
|
case "reconcile":
|
||||||
|
return reconcileCommand(ctx, os.Args[2:])
|
||||||
case "rollback":
|
case "rollback":
|
||||||
return rollbackCommand(ctx, os.Args[2:])
|
return rollbackCommand(ctx, os.Args[2:])
|
||||||
case "prune":
|
case "prune":
|
||||||
@@ -62,7 +66,7 @@ func run() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
func usage() error {
|
func usage() error {
|
||||||
return errors.New("usage: tend <check|package|push|receive|check-server|deploy|status|rollback|prune|version> [options]")
|
return errors.New("usage: tend <check|package|inspect|push|receive|check-server|deploy|status|reconcile|rollback|prune|version> [options]")
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkCommand(args []string) error {
|
func checkCommand(args []string) error {
|
||||||
@@ -146,6 +150,39 @@ func deployCommand(ctx context.Context, args []string) error {
|
|||||||
return writeJSON(report)
|
return writeJSON(report)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func inspectCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("inspect", flag.ContinueOnError)
|
||||||
|
set.SetOutput(os.Stderr)
|
||||||
|
path := set.String("config", "", "target configuration")
|
||||||
|
artifact := set.String("artifact", "", "release archive")
|
||||||
|
sha := set.String("sha256", "", "expected artifact SHA-256")
|
||||||
|
approved := set.String("approve-sha256", "", "separately reviewed artifact SHA-256")
|
||||||
|
if err := set.Parse(args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if set.NArg() != 0 {
|
||||||
|
return errors.New("inspect accepts no positional arguments")
|
||||||
|
}
|
||||||
|
cfg, _, err := loadConfig(*path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
artifactAbs, err := filepath.Abs(*artifact)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
report, err := newManager().Deploy(ctx, cfg, deploy.Request{
|
||||||
|
Artifact: artifactAbs,
|
||||||
|
SHA256: *sha,
|
||||||
|
ApprovedSHA256: *approved,
|
||||||
|
Activate: false,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return writeJSON(report)
|
||||||
|
}
|
||||||
|
|
||||||
func pushCommand(ctx context.Context, args []string) error {
|
func pushCommand(ctx context.Context, args []string) error {
|
||||||
set := flag.NewFlagSet("push", flag.ContinueOnError)
|
set := flag.NewFlagSet("push", flag.ContinueOnError)
|
||||||
set.SetOutput(os.Stderr)
|
set.SetOutput(os.Stderr)
|
||||||
@@ -249,6 +286,27 @@ func statusCommand(ctx context.Context, args []string) error {
|
|||||||
}
|
}
|
||||||
return writeJSON(result)
|
return writeJSON(result)
|
||||||
}
|
}
|
||||||
|
func reconcileCommand(ctx context.Context, args []string) error {
|
||||||
|
set := flag.NewFlagSet("reconcile", flag.ContinueOnError)
|
||||||
|
set.SetOutput(os.Stderr)
|
||||||
|
path := set.String("config", "", "absolute target configuration")
|
||||||
|
_ = set.Bool("json", false, "emit the reconciliation report as JSON")
|
||||||
|
if err := set.Parse(args); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if set.NArg() != 0 {
|
||||||
|
return errors.New("reconcile accepts no positional arguments")
|
||||||
|
}
|
||||||
|
cfg, _, err := loadConfig(*path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
report, err := newManager().Reconcile(ctx, cfg)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return writeJSON(report)
|
||||||
|
}
|
||||||
func rollbackCommand(ctx context.Context, args []string) error {
|
func rollbackCommand(ctx context.Context, args []string) error {
|
||||||
set := flag.NewFlagSet("rollback", flag.ContinueOnError)
|
set := flag.NewFlagSet("rollback", flag.ContinueOnError)
|
||||||
set.SetOutput(os.Stderr)
|
set.SetOutput(os.Stderr)
|
||||||
|
|||||||
+18
-4
@@ -49,8 +49,12 @@ Blue/green mode points the inactive slot at the new release, restarts and
|
|||||||
probes it, atomically replaces one imported Caddy handler, validates the full
|
probes it, atomically replaces one imported Caddy handler, validates the full
|
||||||
Caddy configuration, reloads Caddy, and records the prior active slot.
|
Caddy configuration, reloads Caddy, and records the prior active slot.
|
||||||
|
|
||||||
Singleton mode starts the new release in a hardened transient systemd unit on a
|
Singleton mode starts the target release in a hardened transient systemd unit on a
|
||||||
separate loopback address and probes it. Tend then validates and atomically
|
separate loopback address and probes it. The unit is named from the service and
|
||||||
|
a fresh bounded operation ID; a separate strict candidate-lease file binds that
|
||||||
|
unit to the candidate release, address, and start time without changing the
|
||||||
|
schema-1 deployment-state contract used by retained recovery binaries. Tend
|
||||||
|
then validates and atomically
|
||||||
routes the imported Caddy handler to that candidate. While the candidate serves
|
routes the imported Caddy handler to that candidate. While the candidate serves
|
||||||
the canonical origin, Tend changes the current release pointer, restarts and
|
the canonical origin, Tend changes the current release pointer, restarts and
|
||||||
probes the installed fixed-address unit, validates Caddy again, and routes back
|
probes the installed fixed-address unit, validates Caddy again, and routes back
|
||||||
@@ -69,6 +73,13 @@ over the recorded state. It rechecks local health/readiness and public reachabil
|
|||||||
but deliberately does not apply a future release's content marker to an older
|
but deliberately does not apply a future release's content marker to an older
|
||||||
release whose routes may differ. Pruning preserves both active and previous releases.
|
release whose routes may differ. Pruning preserves both active and previous releases.
|
||||||
|
|
||||||
|
`tend reconcile` is a read-only observation boundary. It compares persisted
|
||||||
|
state, systemd activity, current/previous release pointers, and exact handler
|
||||||
|
file bytes. It does not claim to inspect Caddy's currently loaded in-memory
|
||||||
|
configuration, and it never clears or stops a candidate. The report makes
|
||||||
|
retained, inactive, ambiguous, and settled states legible before a future
|
||||||
|
explicit recovery operation is approved.
|
||||||
|
|
||||||
## Evidence boundary
|
## Evidence boundary
|
||||||
|
|
||||||
Every attempted activation and explicit rollback emits bounded, versioned JSONL
|
Every attempted activation and explicit rollback emits bounded, versioned JSONL
|
||||||
@@ -76,5 +87,8 @@ events with an operation ID, service, approved artifact digest, source commit,
|
|||||||
release version, phase, slot, elapsed duration, and outcome. Values are
|
release version, phase, slot, elapsed duration, and outcome. Values are
|
||||||
validated rather than copied from command output. The log contains no
|
validated rather than copied from command output. The log contains no
|
||||||
environment values, arbitrary process output, HTTP bodies, or secret paths.
|
environment values, arbitrary process output, HTTP bodies, or secret paths.
|
||||||
Identity, entropy, file, and downstream observability failures are deliberately
|
Release-identity, event-file, and downstream observability failures are
|
||||||
best effort and cannot control Tend's deployment or rollback result.
|
deliberately best effort and cannot control Tend's deployment or rollback
|
||||||
|
result. A fresh operation identity is operationally required for a singleton
|
||||||
|
candidate unit; entropy failure therefore stops either forward activation or
|
||||||
|
rollback before a candidate process or traffic change.
|
||||||
|
|||||||
@@ -5,6 +5,67 @@ general reliability claim. It preserves each dated campaign and its limitations
|
|||||||
instead of rewriting earlier observations as though later fixes had already
|
instead of rewriting earlier observations as though later fixes had already
|
||||||
existed.
|
existed.
|
||||||
|
|
||||||
|
## Operation-scoped lease compatibility campaign — August 18, 2026
|
||||||
|
|
||||||
|
The candidate-lease compatibility fix completed trusted verification, exact
|
||||||
|
candidate reproduction, live activation, rollback, reactivation, and a bounded
|
||||||
|
failure injection on the production Linux/systemd/Caddy host. Conventional
|
||||||
|
deployment state remained schema 1 throughout. Singleton operation identity
|
||||||
|
used the separate adjacent lease introduced by this candidate.
|
||||||
|
|
||||||
|
### Assessed Tend candidate
|
||||||
|
|
||||||
|
- Private implementation source commit:
|
||||||
|
`789976ce766fd457ca10762540135e4e0e74cfe3`.
|
||||||
|
- Version: `v0.2.0-preview.2`.
|
||||||
|
- Linux/amd64 binary SHA-256:
|
||||||
|
`d0109bf037e493c7a046defe37818c3c1811806360b99c9a0910213665bd95c5`.
|
||||||
|
- Release-candidate archive SHA-256:
|
||||||
|
`2c225e0b9dd0be2d36fda62ab8d0b52cd9b28f9336c60dfb5edf3e715f450f95`.
|
||||||
|
- Toolchain: Go 1.26.6, `CGO_ENABLED=0`, `-trimpath`.
|
||||||
|
- Trusted verification run 366 and release-candidate run 369 passed for the
|
||||||
|
exact commit. The release workflow built the archive twice with identical
|
||||||
|
bytes and published checksums, build metadata, and an SPDX SBOM.
|
||||||
|
- The forge upload action reported a finalized 3.18 MB artifact, but both
|
||||||
|
documented artifact-list endpoints returned an empty result. A clean
|
||||||
|
independent clone therefore built the package twice and reproduced the
|
||||||
|
workflow's exact archive digest before deployment. This is recorded as an
|
||||||
|
artifact-publication limitation, not described as a successful consumer
|
||||||
|
download.
|
||||||
|
- The previous installed Tend binary remained retained by checksum before the
|
||||||
|
candidate replaced the active tool. Server policy validation then passed.
|
||||||
|
|
||||||
|
### Live maintenance and failure evidence
|
||||||
|
|
||||||
|
Gamertan activated archive
|
||||||
|
`c17b4db1a4e2fd5406b392ec72195b947272097e4d1bfcec9d700d0889c3a4c6`,
|
||||||
|
rolled back to its recorded previous archive, and reactivated the first archive.
|
||||||
|
Both blue/green units remained active with zero restarts. Reconciliation was
|
||||||
|
settled after every transition.
|
||||||
|
|
||||||
|
The Sandwich Hime website performed the same sequence with archive
|
||||||
|
`d418f93ced3307fa788f5d2209b5f09f16bceabec4a9698a9beec6a05e439f35`.
|
||||||
|
Every successful forward and rollback operation removed its operation-scoped
|
||||||
|
candidate lease and transient unit. A final intentional local-smoke failure
|
||||||
|
asked the otherwise valid candidate for an impossible marker. The candidate
|
||||||
|
failed before Caddy or release-pointer mutation, stopped, removed its lease,
|
||||||
|
recorded the failed attempt without changing the active release, and reconciled
|
||||||
|
as settled and consistent.
|
||||||
|
|
||||||
|
After the campaign, Gamertan, its news and case-study indexes, Sandwich Hime,
|
||||||
|
its documentation and tutorial, the Gamertan-mounted Sandwich route, and EQL
|
||||||
|
health all returned HTTP 200. Caddy, both Gamertan slots, and the installed
|
||||||
|
Sandwich Hime service were active with zero restarts and no warning-or-higher
|
||||||
|
journal entries during the campaign.
|
||||||
|
|
||||||
|
Observatory is not claimed by this candidate campaign. Its current dogfood
|
||||||
|
deployment is a Docker Compose singleton, which remains outside Tend's
|
||||||
|
versioned systemd/Caddy strategies. The earlier three-service campaign remains
|
||||||
|
valid for its assessed candidate, but the current commit must not be tagged
|
||||||
|
until Observatory either returns to a supported topology or a separately
|
||||||
|
reviewed Compose strategy completes the same activation, rollback, and failure
|
||||||
|
gates.
|
||||||
|
|
||||||
## Final Preview 2 code-candidate campaign — August 18, 2026
|
## Final Preview 2 code-candidate campaign — August 18, 2026
|
||||||
|
|
||||||
The final v0.2 Preview 2 implementation candidate completed two explicit
|
The final v0.2 Preview 2 implementation candidate completed two explicit
|
||||||
|
|||||||
+341
-13
@@ -1,6 +1,6 @@
|
|||||||
# Dogfood friction ledger
|
# Dogfood friction ledger
|
||||||
|
|
||||||
This document records friction observed through August 18, 2026 while using
|
This document records friction observed through August 19, 2026 while using
|
||||||
Tend for real maintenance releases. Friction is evidence about the product: it
|
Tend for real maintenance releases. Friction is evidence about the product: it
|
||||||
should become either a clearer contract, safer automation, or an explicit
|
should become either a clearer contract, safer automation, or an explicit
|
||||||
non-goal. It must not become application-specific shell lore.
|
non-goal. It must not become application-specific shell lore.
|
||||||
@@ -22,6 +22,10 @@ A Tend workflow should make the secure path the short, documented path:
|
|||||||
- first installation, maintenance, activation, rollback, and pruning are
|
- first installation, maintenance, activation, rollback, and pruning are
|
||||||
distinct operations;
|
distinct operations;
|
||||||
- every mutation has a recorded prior state and a tested restoration path;
|
- every mutation has a recorded prior state and a tested restoration path;
|
||||||
|
- Tend-owned state is an evidence journal and reconciliation aid, not an
|
||||||
|
exclusive claim over an otherwise conventional systemd/Caddy service;
|
||||||
|
- tool-owned transient resources are leased, inspectable, resumable, and safe
|
||||||
|
to reconcile without requiring operators to understand internal names;
|
||||||
- unrelated services may build and prepare candidates concurrently, while the
|
- unrelated services may build and prepare candidates concurrently, while the
|
||||||
shared Caddy activation boundary remains serialized.
|
shared Caddy activation boundary remains serialized.
|
||||||
|
|
||||||
@@ -32,14 +36,23 @@ A Tend workflow should make the secure path the short, documented path:
|
|||||||
| F-01 | Workflow mitigation | CI packaging needs proof that the exact commit was pushed. | Packaging verifies the configured remote and fails if Git cannot authenticate. The trusted workflow retains only its job-scoped read-only checkout credential. | Make the proof credential or an authenticated source attestation an explicit Tend input and evidence boundary. |
|
| F-01 | Workflow mitigation | CI packaging needs proof that the exact commit was pushed. | Packaging verifies the configured remote and fails if Git cannot authenticate. The trusted workflow retains only its job-scoped read-only checkout credential. | Make the proof credential or an authenticated source attestation an explicit Tend input and evidence boundary. |
|
||||||
| F-02 | Open | Application-owned unit and configuration changes are outside the binary activation transaction. | Operators stage, validate, back up, and restore those files separately. | Add an allowlisted configuration transaction or record and validate exact configuration digests. |
|
| F-02 | Open | Application-owned unit and configuration changes are outside the binary activation transaction. | Operators stage, validate, back up, and restore those files separately. | Add an allowlisted configuration transaction or record and validate exact configuration digests. |
|
||||||
| F-03 | Open | A singleton candidate may not reproduce the installed unit's arguments and application configuration. | It receives the production environment file and an isolated listen override; activation still fails closed. | Add a bounded application preflight and explicit, validated candidate invocation. |
|
| F-03 | Open | A singleton candidate may not reproduce the installed unit's arguments and application configuration. | It receives the production environment file and an isolated listen override; activation still fails closed. | Add a bounded application preflight and explicit, validated candidate invocation. |
|
||||||
| F-04 | Partially resolved | Artifact production, review, approval, transfer, and activation require several identity checks. | v0.2 standardizes the evidence bundle and restricted transfer; the operator still repeats the approved digest intentionally. | Add offline inspection and approval ergonomics without combining build authority and production authority. |
|
| F-04 | Partially resolved in development | Artifact production, review, approval, transfer, and activation require several identity checks. | v0.2 standardizes the evidence bundle and restricted transfer; `tend inspect` now exposes the complete non-mutating artifact validation as a named workflow. | Preserve explicit approval while adding digest-bound publication receipts. |
|
||||||
| F-05 | Partially resolved | Tend maintenance assumes an existing adopted service and current release. | `check-server` validates a prepared host, while first installation remains a separately reviewed operator procedure. | Define an explicit `install` or `adopt` transaction rather than silently treating bootstrap as maintenance. |
|
| F-05 | Partially resolved | Tend maintenance assumes an existing adopted service and current release. | `check-server` validates a prepared host, while first installation remains a separately reviewed operator procedure. | Define an explicit `install` or `adopt` transaction rather than silently treating bootstrap as maintenance. |
|
||||||
| F-06 | Partially resolved | Binary rollback can be unsafe when service configuration has changed incompatibly. | v0.2 records desired, candidate, active, previous, and last-attempt releases, but external configuration compatibility is operator-owned. | Bind non-secret configuration identities and preflight results to release state. |
|
| F-06 | Partially resolved | Binary rollback can be unsafe when service configuration has changed incompatibly. | v0.2 records desired, candidate, active, previous, and last-attempt releases, but external configuration compatibility is operator-owned. | Bind non-secret configuration identities and preflight results to release state. |
|
||||||
| F-07 | Open | `GOPROXY=off` does not prove that every module metadata lookup is available locally. | Packaging stops before artifact creation when Go cannot resolve the complete pinned module graph. | Separate checksum-verified dependency resolution from a network-disabled, cache-completeness-checked build stage. |
|
| F-07 | Open | `GOPROXY=off` does not prove that every module metadata lookup is available locally. | Packaging stops before artifact creation when Go cannot resolve the complete pinned module graph. | Separate checksum-verified dependency resolution from a network-disabled, cache-completeness-checked build stage. |
|
||||||
| F-08 | Open | Packaging an otherwise clean pushed commit from a linked Git worktree fails Go's required VCS-status stamp. | Tend stops before artifact creation; package the same exact commit from a clean standalone clone. | Detect linked worktrees during `check`/`package`, explain the supported source shape, and assess a provenance-preserving worktree build that does not weaken `-buildvcs=true`. |
|
| F-08 | Partially resolved in development | Packaging an otherwise clean pushed commit from a linked Git worktree fails Go's required VCS-status stamp. | Tend now detects a linked worktree before remote proof and build work and gives an exact standalone-clone instruction. | Assess provenance-preserving linked-worktree support without weakening `-buildvcs=true`. |
|
||||||
| F-09 | Resolved in Preview 2 | Restarting a fixed-address singleton briefly exposed Caddy to an unavailable upstream. | Tend now routes the imported handler to the candidate, restarts and proves the installed unit behind that handoff, then routes back while the candidate remains healthy through the activation window. | Preserve the production regression campaign and failure-injection matrix. |
|
| F-09 | Resolved in Preview 2 | Restarting a fixed-address singleton briefly exposed Caddy to an unavailable upstream. | Tend now routes the imported handler to the candidate, restarts and proves the installed unit behind that handoff, then routes back while the candidate remains healthy through the activation window. | Preserve the production regression campaign and failure-injection matrix. |
|
||||||
| F-10 | Resolved in Preview 2 | A hardened root umask could make an extracted application binary executable only by root. | Tend reapplies every validated archive mode explicitly; extraction is tested under umask `0077`. | Preserve the mode and non-root candidate tests. |
|
| F-10 | Resolved in Preview 2 | A hardened root umask could make an extracted application binary executable only by root. | Tend reapplies every validated archive mode explicitly; extraction is tested under umask `0077`. | Preserve the mode and non-root candidate tests. |
|
||||||
| F-11 | Evidence practice | A single external observer can report common-mode client or network errors as apparent multi-service downtime. | Production campaigns retain server state and use an independent observer before classifying a continuity failure. | Standardize multi-vantage continuity evidence without making an observability dependency part of deployment authority. |
|
| F-11 | Evidence practice | A single external observer can report common-mode client or network errors as apparent multi-service downtime. | Production campaigns retain server state and use an independent observer before classifying a continuity failure. | Standardize multi-vantage continuity evidence without making an observability dependency part of deployment authority. |
|
||||||
|
| F-12 | Partially resolved in development | A deliberately retained singleton handoff candidate could occupy Tend's fixed transient-unit name and block the next valid activation. | Candidates now have operation-scoped unit identities and bounded persisted leases; `reconcile --json` reports release identity, unit, pointer, and handler-file facts without mutation. | Add an explicit, idempotent `resume`/repair operation only after its route-identity and failure-injection model is proven. |
|
||||||
|
| F-13 | Open | A configured public-origin probe may resolve to a different deployment after a DNS or edge migration. | Operators separately verify local routed-origin identity and external DNS topology before activation. | Bind probes to expected release and edge identities; classify topology drift instead of treating a body marker as deployment proof. |
|
||||||
|
| F-14 | Open | An application can bind durable data identity to an absolute path that a binary-only candidate never exercises. | Preserve the production path and run application-owned validation before traffic moves. | Add a bounded, explicitly read-only application preflight and record which configuration/data identities it assessed. |
|
||||||
|
| F-15 | Product direction | Requiring Tend-specific residue and metadata to be perfect can make the facilitator feel like an exclusive deployment owner. | Conventional systemd/Caddy recovery remains authoritative and every manual intervention is captured as evidence. | Make state append-only and migratable, infer observed state safely, and keep Tend removable without making the service obscure or undeployable. |
|
||||||
|
| F-16 | Open | A reusable Docker network alias can identify both the live service and a retained handoff candidate, making the routed target ambiguous during activation. | Pin the temporary Caddy handoff to the exact, observed candidate address and restore the reviewed application handler after activation. | Allocate operation-scoped network identities and reject any candidate or handler target that resolves to more than one container or release identity. |
|
||||||
|
| F-17 | Open | A CI job can report a successful artifact upload while the forge artifact API exposes no retrievable artifact to the approval/deployment client. | Reproduce and verify the exact pushed commit locally, then compare two packages byte-for-byte before approving the digest; never guess an artifact URL. | Define a digest-bound artifact handoff with an independently readable receipt and fail the workflow unless the approval client can retrieve and inspect the exact uploaded bytes. |
|
||||||
|
| F-18 | Open | Tend's current strategies do not model a Docker Compose singleton, so a manual Observatory replacement briefly exposed an unavailable upstream. | The exact image, data backup, rollback image, and health probes were preserved, but Compose replaced the only live container and an external observer saw a bounded `502`/`503` window. | Design an explicit Compose strategy with operation-scoped candidates, unambiguous routing, durable-data preflight, activation continuity, and automatic rollback; do not add arbitrary container hooks to the systemd strategies. |
|
||||||
|
| F-19 | Open | A successful application schema migration can make the previous binary unreadable even when its image and service definition are intact. | Preserve verified database backups and prefer completing the proven forward activation; do not execute a binary-only rollback after a forward-only migration. | Make applications declare schema compatibility and an explicit data restoration or forward-recovery plan before Tend offers automatic rollback. |
|
||||||
|
| F-20 | Open | A stateful migration can require bounded temporary scratch space that a stateless binary candidate never exercises. | Run the exact candidate against a copied production data set under its proposed resource limits; record the reviewed scratch budget in the service definition. | Bind resource/configuration digests to the candidate and add an application-owned stateful preflight contract without arbitrary hooks. |
|
||||||
|
|
||||||
## F-01: pushed-commit proof in CI
|
## F-01: pushed-commit proof in CI
|
||||||
|
|
||||||
@@ -307,20 +320,335 @@ is closed. A Linux regression test sets umask `0077`, extracts the release, and
|
|||||||
requires the installed binary to remain executable by the service identity.
|
requires the installed binary to remain executable by the service identity.
|
||||||
The successful maintenance campaign used that corrected Tend binary.
|
The successful maintenance campaign used that corrected Tend binary.
|
||||||
|
|
||||||
|
## F-12: retained-candidate reconciliation
|
||||||
|
|
||||||
|
### Observation
|
||||||
|
|
||||||
|
An Observatory activation failure intentionally left its proven handoff
|
||||||
|
candidate routed and running for operator recovery. A later push transferred
|
||||||
|
and validated a new content-addressed artifact, then stopped before candidate
|
||||||
|
startup because systemd still had Tend's fixed candidate unit loaded:
|
||||||
|
|
||||||
|
`Unit observatory-tend-candidate.service was already loaded or has a fragment file.`
|
||||||
|
|
||||||
|
The artifact, application, and host were valid. The collision was Tend-owned
|
||||||
|
residue from its own safe fallback behavior. Recovery required starting the
|
||||||
|
same stateless fallback under an independently named hardened unit, proving it,
|
||||||
|
routing Caddy to it, and only then stopping the old candidate and confirming
|
||||||
|
its port and unit name were free. No accepted application state was lost, but
|
||||||
|
the operator had to understand Tend's internal unit convention.
|
||||||
|
|
||||||
|
### Implemented foundation
|
||||||
|
|
||||||
|
- Transient candidate units are named by service plus the first 12 hexadecimal
|
||||||
|
characters of a fresh 128-bit operation identity.
|
||||||
|
- Conventional deployment state remains schema 1 so the retained v0.1 binary
|
||||||
|
can still read active and previous release identities. Singleton operations
|
||||||
|
write a separate strict, adjacent candidate-lease file containing the
|
||||||
|
operation, release, unit, address, and start time.
|
||||||
|
- `tend reconcile --json` reports persisted state, installed/candidate unit
|
||||||
|
activity, release pointers, and whether the exact handler file matches the
|
||||||
|
installed or candidate upstream. It explicitly distinguishes handler file
|
||||||
|
bytes from Caddy's loaded runtime configuration and performs no mutation.
|
||||||
|
- A retained forward-activation or rollback candidate lease survives an
|
||||||
|
incomplete recovery instead of being erased by a generic failed-attempt path.
|
||||||
|
Older binaries ignore the additive lease file, so operators must reconcile
|
||||||
|
before downgrading or attempting another activation.
|
||||||
|
|
||||||
|
### Remaining options to assess
|
||||||
|
|
||||||
|
- Extend the candidate lease with separately observed health and loaded-route
|
||||||
|
evidence rather than inferring either from a successful file write.
|
||||||
|
- Add an explicit `tend resume` operation that distinguishes active, routed,
|
||||||
|
rollback, abandoned, and unknown candidates without mutating by default.
|
||||||
|
- Permit automatic cleanup only after Tend proves that Caddy, current/previous
|
||||||
|
pointers, and the installed unit do not reference the candidate and that a
|
||||||
|
healthy route remains.
|
||||||
|
- Make a repeated activation of the same approved digest idempotently resume
|
||||||
|
the recorded operation instead of restaging or colliding with itself.
|
||||||
|
|
||||||
|
Never resolve the collision by blindly stopping the loaded unit. A retained
|
||||||
|
candidate may be the only healthy route after a failed singleton activation.
|
||||||
|
|
||||||
|
## F-13: public-origin topology and release identity
|
||||||
|
|
||||||
|
### Observation
|
||||||
|
|
||||||
|
During the same recovery, the public DNS record had already moved to a new
|
||||||
|
edge while the old host's Tend policy still named the canonical HTTPS origin
|
||||||
|
as its post-activation smoke target. A request from the old host would therefore
|
||||||
|
test a different deployment. It could fail even when the old host was healthy,
|
||||||
|
or pass against a matching marker served by the new host. Neither result proves
|
||||||
|
the release that Tend just activated.
|
||||||
|
|
||||||
|
### Options to assess
|
||||||
|
|
||||||
|
- Separate a local routed-origin probe—Caddy with the canonical Host/SNI on the
|
||||||
|
deployment host—from an external DNS-origin observation.
|
||||||
|
- Bind local success to a non-secret application release identity such as the
|
||||||
|
approved artifact digest, commit, and version, not only a human page marker.
|
||||||
|
- Record the expected public edge identity or resolved address set at approval
|
||||||
|
time. If it changes, classify the result as topology drift and require an
|
||||||
|
explicit migration decision instead of reporting an application failure.
|
||||||
|
- Allow independent external observers as additional evidence, but never let
|
||||||
|
a response from an unidentified deployment authorize activation.
|
||||||
|
|
||||||
|
Public reachability remains valuable; it must be evidence about the intended
|
||||||
|
deployment rather than merely evidence that the hostname answered.
|
||||||
|
|
||||||
|
## F-14: application data-path and preflight fidelity
|
||||||
|
|
||||||
|
### Observation
|
||||||
|
|
||||||
|
An exact, checksummed Observatory data copy was mounted at a different absolute
|
||||||
|
path in a container. Raw objects and durable SQLite databases were byte
|
||||||
|
identical, but the application catalogue intentionally compared stored segment
|
||||||
|
paths with filesystem-derived identities and failed closed. Preserving the
|
||||||
|
original in-container data path made the same Preview 15 binary ready in four
|
||||||
|
seconds with zero restarts and bounded memory.
|
||||||
|
|
||||||
|
This was an application portability constraint, not a reason for Tend to
|
||||||
|
rewrite database state. It also demonstrates the limit of a binary-only
|
||||||
|
stateless candidate: executable health cannot prove compatibility with the
|
||||||
|
real configuration, credentials, mounts, or durable data.
|
||||||
|
|
||||||
|
SQLite `-wal`, `-shm`, and process-lock files also changed during ordinary
|
||||||
|
open/close behavior while raw segments and durable databases did not. Migration
|
||||||
|
evidence should distinguish durable application truth from ephemeral runtime
|
||||||
|
coordination files.
|
||||||
|
|
||||||
|
### Options to assess
|
||||||
|
|
||||||
|
- Support a fixed, bounded application-owned preflight argument vector with an
|
||||||
|
explicit non-mutating contract, timeout, output limit, and no shell.
|
||||||
|
- Record the exact non-secret configuration digest, data-root identity, mount
|
||||||
|
identity, and checks performed alongside binary candidate evidence.
|
||||||
|
- Let applications define their durable migration evidence set; Tend should
|
||||||
|
transport and report those digests but must not infer database semantics or
|
||||||
|
edit state.
|
||||||
|
- Require rollback compatibility to be assessed against the same configuration
|
||||||
|
and data identities before an older binary is called safe.
|
||||||
|
|
||||||
|
## F-15: facilitator rather than owner
|
||||||
|
|
||||||
|
Tend's strict boundaries are valuable where authority changes hands: source
|
||||||
|
provenance, artifact digest approval, restricted transport, secret references,
|
||||||
|
path validation, traffic movement, and rollback. Strictness becomes harmful
|
||||||
|
when Tend's internal names or stale metadata are treated as application
|
||||||
|
requirements that an authorized maintainer must reverse-engineer.
|
||||||
|
|
||||||
|
The target model is less invasive:
|
||||||
|
|
||||||
|
- systemd units, Caddy handlers, environment files, release directories, and
|
||||||
|
application checks remain conventional and independently operable;
|
||||||
|
- Tend records an append-only migration/deployment journal and derives an
|
||||||
|
observed state before proposing mutation;
|
||||||
|
- schema migrations preserve old records and explain compatibility rather than
|
||||||
|
silently rejecting safe, recognizable state;
|
||||||
|
- `check`, `status`, and `reconcile` show facts and proposed repairs without
|
||||||
|
mutation; approval is required for artifact selection and traffic movement,
|
||||||
|
not for Tend implementation trivia;
|
||||||
|
- installation and adoption are explicit, while uninstalling Tend leaves an
|
||||||
|
understandable, runnable service and complete evidence trail.
|
||||||
|
|
||||||
|
This does not relax hostile-input defenses. It moves strictness to the trust
|
||||||
|
boundary and makes recovery humane for the authenticated operator.
|
||||||
|
|
||||||
|
## F-16: ambiguous container-network identity
|
||||||
|
|
||||||
|
### Observation
|
||||||
|
|
||||||
|
During the Observatory Preview 16 activation, the retained stateless handoff
|
||||||
|
container and the live Compose service both answered to the Docker network
|
||||||
|
alias `observatory`. A Caddy target such as `observatory:8093` could therefore
|
||||||
|
resolve to either release. Removing or restarting one container could also
|
||||||
|
change which release received traffic. The application and both containers
|
||||||
|
were healthy; the ambiguity existed solely in deployment identity.
|
||||||
|
|
||||||
|
The activation used a separately validated Caddy fragment pinned to the exact
|
||||||
|
candidate address, proved its release marker before traffic moved, replaced
|
||||||
|
the live service, and then restored the reviewed application handler. No
|
||||||
|
ambiguous alias was used for the handoff.
|
||||||
|
|
||||||
|
### Options to assess
|
||||||
|
|
||||||
|
- Give every candidate an operation-scoped container name, network alias, and
|
||||||
|
lease that bind directly to its artifact digest and expected address.
|
||||||
|
- Resolve and inspect a proposed upstream immediately before Caddy validation;
|
||||||
|
reject zero, multiple, or identity-mismatched targets.
|
||||||
|
- Prefer a dedicated candidate network or an address supplied by the container
|
||||||
|
runtime over a stable alias shared with the live service.
|
||||||
|
- Record the exact routed target and release proof in activation state, then
|
||||||
|
verify that the restored production handler identifies the active release.
|
||||||
|
|
||||||
|
Do not treat a healthy response from an ambiguous service name as release
|
||||||
|
proof. Availability and deployment identity are separate properties.
|
||||||
|
|
||||||
|
## F-17: CI artifact publication is not artifact availability
|
||||||
|
|
||||||
|
### Observation
|
||||||
|
|
||||||
|
The trusted Observatory release-candidate workflow completed its two builds,
|
||||||
|
byte comparison, checksums, manifest, and SBOM, and its upload action reported
|
||||||
|
success. The forge artifact API subsequently returned no artifact for that
|
||||||
|
run. The deployment client therefore had no independently discoverable object
|
||||||
|
to download and inspect. Guessing a web-interface route or treating a green
|
||||||
|
upload step as possession of the bytes would have weakened the build-approve-
|
||||||
|
deploy boundary.
|
||||||
|
|
||||||
|
The release was instead reproduced twice from a fresh, clean clone of the
|
||||||
|
exact pushed commit with the pinned toolchain. The packages compared
|
||||||
|
byte-for-byte, their manifest, checksums, SBOM, version, commit, and tree were
|
||||||
|
verified, and the approved digest was recorded. This preserved release
|
||||||
|
identity, but it is an operator workaround rather than the desired CI handoff.
|
||||||
|
|
||||||
|
### Options to assess
|
||||||
|
|
||||||
|
- Require the publisher to return a versioned receipt containing forge, run,
|
||||||
|
artifact identifier, size, digest, retention, and retrieval endpoint.
|
||||||
|
- Add a separate read-only verification step that downloads the artifact using
|
||||||
|
the same interface available to the approval client and reruns `tend
|
||||||
|
inspect` before the workflow is considered publish-complete.
|
||||||
|
- Support a content-addressed, append-only artifact store whose object name is
|
||||||
|
the approved SHA-256 and whose credentials remain separate from production.
|
||||||
|
- Let `tend push` accept only a locally present artifact plus an optional
|
||||||
|
verified publication receipt; never allow a successful CI status alone to
|
||||||
|
select bytes for deployment.
|
||||||
|
|
||||||
|
Artifact availability must be proven from the consumer side. A successful
|
||||||
|
upload log is evidence of an attempted publication, not evidence that the
|
||||||
|
approved bytes can be recovered.
|
||||||
|
|
||||||
|
The same failure repeated for Tend release-candidate run 369: the pinned job
|
||||||
|
successfully built identical archives, recorded digest
|
||||||
|
`2c225e0b9dd0be2d36fda62ab8d0b52cd9b28f9336c60dfb5edf3e715f450f95`,
|
||||||
|
and finalized a 3.18 MB upload, while both repository-wide and run-scoped REST
|
||||||
|
artifact listings returned no objects. An independent clean build reproduced
|
||||||
|
the exact digest before dogfooding. This confirms the problem is a reusable
|
||||||
|
forge-to-approval gap rather than an Observatory-specific packaging defect.
|
||||||
|
|
||||||
|
## F-18: Docker Compose is not a systemd singleton
|
||||||
|
|
||||||
|
### Observation
|
||||||
|
|
||||||
|
Observatory Preview 18 was packaged reproducibly and its exact scratch image
|
||||||
|
passed a constrained loopback candidate check. The production service on
|
||||||
|
cliff-mads is a Docker Compose singleton, while Tend's supported strategies are
|
||||||
|
systemd/Caddy blue-green and systemd singleton-candidate. Tend was therefore
|
||||||
|
not used to pretend that an unsupported runtime had received a complete Tend
|
||||||
|
activation proof.
|
||||||
|
|
||||||
|
The manual Compose replacement retained the former image and online database
|
||||||
|
backups and completed health, readiness, route, projection, agent, and log
|
||||||
|
checks. An independent public observer nevertheless recorded `200`, then
|
||||||
|
`502`/`503`, then `200` across an approximately 31.6-second replacement and
|
||||||
|
startup window. No application restart loop or accepted-data loss was
|
||||||
|
observed, but this is not continuous delivery.
|
||||||
|
|
||||||
|
Preview.19 strengthened this finding. The exact candidate passed direct health
|
||||||
|
and readiness, but an activation script treated the first transient routed
|
||||||
|
`503` as final and attempted rollback before the proxy path had settled. The
|
||||||
|
previous binary then correctly rejected the newly migrated control schema and
|
||||||
|
entered a fail-closed restart loop. The operator recovered with the already
|
||||||
|
proven forward candidate while the agent remained paused. This was a bounded,
|
||||||
|
observable failure with intact backups, but it demonstrates that proxy
|
||||||
|
settling and data-schema rollback are different gates and cannot share one
|
||||||
|
generic failure branch.
|
||||||
|
|
||||||
|
### Options to assess
|
||||||
|
|
||||||
|
- Add a distinct, versioned Docker Compose strategy rather than arbitrary
|
||||||
|
command hooks or hidden container behavior inside existing strategies.
|
||||||
|
- Allocate an operation-scoped candidate container and network identity that
|
||||||
|
cannot collide with the production alias.
|
||||||
|
- Validate the exact image digest, non-secret Compose/configuration digest,
|
||||||
|
mounts, durable data identity, and application-owned read-only preflight
|
||||||
|
before moving traffic.
|
||||||
|
- Route Caddy to the proven candidate, replace or promote the production
|
||||||
|
service while the candidate remains healthy, then restore the reviewed live
|
||||||
|
handler only after the final container proves its release identity.
|
||||||
|
- Restore the prior handler and retained image automatically when any
|
||||||
|
activation-window probe fails; keep data rollback an explicit
|
||||||
|
application-owned decision.
|
||||||
|
|
||||||
|
This strategy must remain optional. Tend should facilitate a conventional
|
||||||
|
Compose service without requiring Tend-only container labels, aliases, or
|
||||||
|
state for ordinary operator recovery.
|
||||||
|
|
||||||
|
## F-19: binary rollback is not data rollback
|
||||||
|
|
||||||
|
### Observation
|
||||||
|
|
||||||
|
Observatory Preview.19 migrated its live control database from the schema
|
||||||
|
understood by Preview.18 to schema 11 before activation. When a premature
|
||||||
|
public-origin failure triggered binary rollback, Preview.18 rejected the new
|
||||||
|
schema rather than interpreting unknown state. That fail-closed behavior was
|
||||||
|
correct, but Tend-like orchestration cannot infer from two binary identities
|
||||||
|
whether their durable schemas are mutually readable.
|
||||||
|
|
||||||
|
The exact pre-migration control and projection databases had already been
|
||||||
|
copied, integrity-checked, mode-restricted, and SHA-256 verified. The new
|
||||||
|
candidate had also passed the migration on an isolated copy. Because the
|
||||||
|
migration was forward-valid and the old agent was paused, completing the
|
||||||
|
Preview.19 activation preserved more verified state than restoring the backup.
|
||||||
|
|
||||||
|
### Options to assess
|
||||||
|
|
||||||
|
- Let an application declare a versioned, bounded compatibility statement for
|
||||||
|
current-to-candidate and candidate-to-previous data access.
|
||||||
|
- Treat migration completion as a journalled phase after which automatic
|
||||||
|
binary rollback is permitted only when backward readability was explicitly
|
||||||
|
proven.
|
||||||
|
- Support an application-owned, separately approved data-restoration plan;
|
||||||
|
never infer one from release pointers or run an arbitrary rollback hook.
|
||||||
|
- Keep verified backups and the proven forward candidate until the migration
|
||||||
|
soak closes, even when the previous binary remains retained.
|
||||||
|
|
||||||
|
## F-20: stateful preflight needs its real resource envelope
|
||||||
|
|
||||||
|
### Observation
|
||||||
|
|
||||||
|
The Preview.19 stateless candidate and ordinary tests passed, but building a
|
||||||
|
presence-only SQLite index over a copied production projection failed with
|
||||||
|
`database or disk is full` under the service's 64 MiB `/tmp` tmpfs. The exact
|
||||||
|
same candidate and data passed with a 512 MiB tmpfs and drained all pending raw
|
||||||
|
segments. The projection database was healthy; the one-time index sort needed
|
||||||
|
more bounded scratch space than steady-state operation.
|
||||||
|
|
||||||
|
### Options to assess
|
||||||
|
|
||||||
|
- Record the non-secret service configuration digest and resource envelope
|
||||||
|
beside the artifact digest.
|
||||||
|
- Permit a strict application-owned preflight command selected from a reviewed
|
||||||
|
schema, never an arbitrary shell string.
|
||||||
|
- Exercise migrations against a copied or snapshot-backed production data set
|
||||||
|
under the candidate's exact CPU, memory, temporary-storage, filesystem, and
|
||||||
|
credential boundaries.
|
||||||
|
- Report resource exhaustion distinctly from corrupt data, failed health, or
|
||||||
|
incompatible schema so recovery guidance remains accurate.
|
||||||
|
|
||||||
## Prioritization
|
## Prioritization
|
||||||
|
|
||||||
The recommended implementation order is:
|
The recommended implementation order is:
|
||||||
|
|
||||||
1. final production proof of the singleton traffic handoff;
|
1. operation-scoped candidate leases plus explicit resume/repair built on the
|
||||||
2. a checksum-verified resolver and network-disabled package contract;
|
new read-only reconciliation report;
|
||||||
3. an early linked-worktree diagnostic and a provenance-preserving support
|
2. migration compatibility and stateful-resource preflight contracts;
|
||||||
decision;
|
3. a bounded Docker Compose strategy with operation-scoped container/network
|
||||||
4. `tend inspect` and a standardized CI artifact/approval contract;
|
identities and unambiguous upstream
|
||||||
5. exact release plus configuration identity in status and state;
|
resolution and release proof;
|
||||||
6. a bounded application preflight contract;
|
4. release-bound local routed-origin proof separated from external DNS proof;
|
||||||
7. restricted transfer and receive with host policy;
|
5. a bounded application preflight and configuration/data identity record;
|
||||||
8. explicit adoption for existing services;
|
6. a consumer-verifiable, digest-bound CI artifact publication receipt;
|
||||||
9. managed configuration only after its restoration and failure-injection
|
7. final production proof of the singleton traffic handoff;
|
||||||
|
8. a checksum-verified resolver and network-disabled package contract;
|
||||||
|
9. a provenance-preserving linked-worktree support decision after the new
|
||||||
|
early diagnostic;
|
||||||
|
10. a standardized CI artifact/approval contract building on `tend inspect`;
|
||||||
|
11. exact release plus configuration identity in status and state;
|
||||||
|
12. restricted transfer and receive with host policy;
|
||||||
|
13. explicit adoption for existing services;
|
||||||
|
14. managed configuration only after its restoration and failure-injection
|
||||||
model is as strong as binary activation.
|
model is as strong as binary activation.
|
||||||
|
|
||||||
Friction entries should be updated with the implementing version, tests, and
|
Friction entries should be updated with the implementing version, tests, and
|
||||||
|
|||||||
@@ -1,8 +1,17 @@
|
|||||||
# Schema 1 to schema 2
|
# Schema 1 to schema 2
|
||||||
|
|
||||||
Schema 2 is intentionally not loaded as schema 1. Keep the installed v0.1 Tend
|
Configuration schema 2 and deployment state are separate versioned contracts.
|
||||||
binary available until the first schema-2 activation and rollback have both
|
The service configuration moves to schema 2, while conventional deployment
|
||||||
been exercised.
|
state deliberately remains schema 1 so the retained v0.1 binary can still read
|
||||||
|
active and previous release identities during recovery.
|
||||||
|
|
||||||
|
Singleton operations write an adjacent, strict
|
||||||
|
`<state-file>.candidate-lease.json` file containing the operation, release,
|
||||||
|
unit, address, and start time. Older binaries ignore that additive file. Do not
|
||||||
|
downgrade or start another deployment while a lease is present: first use the
|
||||||
|
new binary's `tend reconcile --json` report to establish which process and
|
||||||
|
route are healthy. Tend does not silently invent a lease for a legacy
|
||||||
|
interrupted operation.
|
||||||
|
|
||||||
1. Move each configuration to `/etc/tend/services/<service>.json`.
|
1. Move each configuration to `/etc/tend/services/<service>.json`.
|
||||||
2. Set `schema_version` to `2`.
|
2. Set `schema_version` to `2`.
|
||||||
|
|||||||
@@ -24,6 +24,10 @@ binary at `/usr/local/bin/tend`; it does not need Git or Go.
|
|||||||
4. Pin the server host key in a dedicated client file. Do not accept a new key
|
4. Pin the server host key in a dedicated client file. Do not accept a new key
|
||||||
interactively during deployment.
|
interactively during deployment.
|
||||||
5. Run `sudo tend check-server` and inspect the allowlisted service names.
|
5. Run `sudo tend check-server` and inspect the allowlisted service names.
|
||||||
|
6. Run `sudo tend reconcile --config /etc/tend/services/example-site.json
|
||||||
|
--json` before the first maintenance release. A settled report is expected;
|
||||||
|
any retained candidate or unknown handler must be understood before traffic
|
||||||
|
changes. The command is read-only.
|
||||||
|
|
||||||
## Build and approve
|
## Build and approve
|
||||||
|
|
||||||
@@ -35,6 +39,23 @@ The maintainer reads the candidate report and copies the exact approved digest
|
|||||||
into the deployment command. Tend refuses a digest that is merely inferred from
|
into the deployment command. Tend refuses a digest that is merely inferred from
|
||||||
the local file or differs from the produced value.
|
the local file or differs from the produced value.
|
||||||
|
|
||||||
|
Inspect the exact downloaded bytes before transfer. This is a complete,
|
||||||
|
read-only artifact validation and does not stage a candidate:
|
||||||
|
|
||||||
|
```text
|
||||||
|
tend inspect --config /review/example-site.json \
|
||||||
|
--artifact /approved/example-site.tar.gz \
|
||||||
|
--sha256 <produced> --approve-sha256 <reviewed>
|
||||||
|
```
|
||||||
|
|
||||||
|
On the host, compare Tend's journal with the conventional service state before
|
||||||
|
and after activation. Reconciliation reports facts and never stops a unit,
|
||||||
|
rewrites a pointer, or changes Caddy:
|
||||||
|
|
||||||
|
```text
|
||||||
|
sudo tend reconcile --config /etc/tend/services/example-site.json
|
||||||
|
```
|
||||||
|
|
||||||
```text
|
```text
|
||||||
tend push --target tend-deploy@server.example \
|
tend push --target tend-deploy@server.example \
|
||||||
--known-hosts /secure/tend_known_hosts \
|
--known-hosts /secure/tend_known_hosts \
|
||||||
@@ -66,5 +87,12 @@ services share Caddy. Tend does not stop the other application.
|
|||||||
units, pointers, and public origin did not change.
|
units, pointers, and public origin did not change.
|
||||||
- Interrupt a transfer: no release becomes active and the incomplete incoming
|
- Interrupt a transfer: no release becomes active and the incomplete incoming
|
||||||
file is removed when the receiver exits.
|
file is removed when the receiver exits.
|
||||||
|
- Interrupt singleton recovery after the candidate is proven. `tend reconcile
|
||||||
|
--json` must name the operation-scoped candidate unit, report whether it is
|
||||||
|
active and whether the handler file targets it, and perform no stop, restart,
|
||||||
|
reload, pointer, or state mutation.
|
||||||
|
|
||||||
After the soak, prune per service. Active and previous releases remain protected.
|
After the soak, prune per service. Active and previous releases remain protected.
|
||||||
|
If activation fails, run `reconcile` before manual recovery so the retained
|
||||||
|
candidate, route, release pointers, and journal disagreement are preserved in
|
||||||
|
one bounded report.
|
||||||
|
|||||||
+188
-21
@@ -31,6 +31,7 @@ type Report struct {
|
|||||||
ActiveRelease string `json:"active_release,omitempty"`
|
ActiveRelease string `json:"active_release,omitempty"`
|
||||||
PreviousRelease string `json:"previous_release,omitempty"`
|
PreviousRelease string `json:"previous_release,omitempty"`
|
||||||
EventWarnings int `json:"event_warnings,omitempty"`
|
EventWarnings int `json:"event_warnings,omitempty"`
|
||||||
|
LeaseCleanupPending bool `json:"lease_cleanup_pending,omitempty"`
|
||||||
}
|
}
|
||||||
type Status struct {
|
type Status struct {
|
||||||
State *state.Record `json:"state,omitempty"`
|
State *state.Record `json:"state,omitempty"`
|
||||||
@@ -38,6 +39,11 @@ type Status struct {
|
|||||||
StateInitialized bool `json:"state_initialized"`
|
StateInitialized bool `json:"state_initialized"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type retainedCandidateError struct{ cause error }
|
||||||
|
|
||||||
|
func (e *retainedCandidateError) Error() string { return e.cause.Error() }
|
||||||
|
func (e *retainedCandidateError) Unwrap() error { return e.cause }
|
||||||
|
|
||||||
type Manager struct {
|
type Manager struct {
|
||||||
Operator Operator
|
Operator Operator
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
@@ -86,6 +92,9 @@ func (m Manager) Deploy(ctx context.Context, cfg config.Config, request Request)
|
|||||||
operationID = ""
|
operationID = ""
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if cfg.Deployment.Strategy == "singleton_candidate" && operationID == "" {
|
||||||
|
return Report{}, errors.New("singleton activation requires a fresh operation identity")
|
||||||
|
}
|
||||||
emit := func(phase, slot, outcome string) {
|
emit := func(phase, slot, outcome string) {
|
||||||
if m.AppendEvent == nil || identityErr != nil || operationID == "" {
|
if m.AppendEvent == nil || identityErr != nil || operationID == "" {
|
||||||
return
|
return
|
||||||
@@ -99,6 +108,17 @@ func (m Manager) Deploy(ctx context.Context, cfg config.Config, request Request)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return Report{}, err
|
return Report{}, err
|
||||||
}
|
}
|
||||||
|
leasePath := state.CandidateLeasePath(cfg.Deployment.StateFile)
|
||||||
|
var candidateLease state.CandidateLease
|
||||||
|
if cfg.Deployment.Strategy == "singleton_candidate" {
|
||||||
|
_, exists, leaseErr := loadCandidateLease(cfg)
|
||||||
|
if leaseErr != nil {
|
||||||
|
return Report{}, leaseErr
|
||||||
|
}
|
||||||
|
if record.CandidateRelease != "" || exists {
|
||||||
|
return Report{}, errors.New("singleton candidate lease is unresolved; run tend reconcile --json before another activation")
|
||||||
|
}
|
||||||
|
}
|
||||||
attemptAt := m.Now().UTC().Format(time.RFC3339)
|
attemptAt := m.Now().UTC().Format(time.RFC3339)
|
||||||
record.DesiredRelease = release
|
record.DesiredRelease = release
|
||||||
record.CandidateRelease = release
|
record.CandidateRelease = release
|
||||||
@@ -106,24 +126,52 @@ func (m Manager) Deploy(ctx context.Context, cfg config.Config, request Request)
|
|||||||
record.LastAttemptOutcome = "running"
|
record.LastAttemptOutcome = "running"
|
||||||
record.LastAttemptAt = attemptAt
|
record.LastAttemptAt = attemptAt
|
||||||
record.UpdatedAt = attemptAt
|
record.UpdatedAt = attemptAt
|
||||||
|
if cfg.Deployment.Strategy == "singleton_candidate" {
|
||||||
|
candidateUnit, unitErr := singletonCandidateUnit(cfg.Service.Name, operationID)
|
||||||
|
if unitErr != nil {
|
||||||
|
return Report{}, unitErr
|
||||||
|
}
|
||||||
|
candidateLease = state.CandidateLease{SchemaVersion: state.CandidateLeaseSchemaVersion, Service: cfg.Service.Name, OperationID: operationID, Release: release, Unit: candidateUnit, Address: cfg.Deployment.Singleton.CandidateAddress, StartedAt: attemptAt}
|
||||||
|
}
|
||||||
if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil {
|
if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil {
|
||||||
return Report{}, err
|
return Report{}, err
|
||||||
}
|
}
|
||||||
|
if cfg.Deployment.Strategy == "singleton_candidate" {
|
||||||
|
if err := state.StoreCandidateLease(leasePath, cfg.Deployment.Root, candidateLease); err != nil {
|
||||||
|
failed := record
|
||||||
|
clearCandidateLease(&failed)
|
||||||
|
failed.LastAttemptOutcome = "failed"
|
||||||
|
failed.UpdatedAt = m.Now().UTC().Format(time.RFC3339)
|
||||||
|
if storeErr := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, failed); storeErr != nil {
|
||||||
|
return Report{}, errors.Join(err, storeErr)
|
||||||
|
}
|
||||||
|
return Report{}, err
|
||||||
|
}
|
||||||
|
}
|
||||||
emit("candidate", inactiveSlot(cfg, record), "running")
|
emit("candidate", inactiveSlot(cfg, record), "running")
|
||||||
switch cfg.Deployment.Strategy {
|
switch cfg.Deployment.Strategy {
|
||||||
case "blue_green":
|
case "blue_green":
|
||||||
err = m.deployBlueGreen(ctx, cfg, record, release)
|
err = m.deployBlueGreen(ctx, cfg, record, release)
|
||||||
case "singleton_candidate":
|
case "singleton_candidate":
|
||||||
err = m.deploySingleton(ctx, cfg, record, release)
|
err = m.deploySingleton(ctx, cfg, record, release, candidateLease.Unit)
|
||||||
default:
|
default:
|
||||||
err = errors.New("unsupported strategy")
|
err = errors.New("unsupported strategy")
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
failed := record
|
failed := record
|
||||||
failed.CandidateRelease = ""
|
var retained *retainedCandidateError
|
||||||
|
if !errors.As(err, &retained) {
|
||||||
|
if cleanupErr := state.RemoveCandidateLease(leasePath); cleanupErr != nil {
|
||||||
|
err = errors.Join(err, fmt.Errorf("candidate lease cleanup failed: %w", cleanupErr))
|
||||||
|
} else {
|
||||||
|
clearCandidateLease(&failed)
|
||||||
|
}
|
||||||
|
}
|
||||||
failed.LastAttemptOutcome = "failed"
|
failed.LastAttemptOutcome = "failed"
|
||||||
failed.UpdatedAt = m.Now().UTC().Format(time.RFC3339)
|
failed.UpdatedAt = m.Now().UTC().Format(time.RFC3339)
|
||||||
_ = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, failed)
|
if storeErr := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, failed); storeErr != nil {
|
||||||
|
err = errors.Join(err, storeErr)
|
||||||
|
}
|
||||||
emit("activation", inactiveSlot(cfg, record), "failed")
|
emit("activation", inactiveSlot(cfg, record), "failed")
|
||||||
return Report{EventWarnings: eventWarnings}, err
|
return Report{EventWarnings: eventWarnings}, err
|
||||||
}
|
}
|
||||||
@@ -132,7 +180,40 @@ func (m Manager) Deploy(ctx context.Context, cfg config.Config, request Request)
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return Report{}, err
|
return Report{}, err
|
||||||
}
|
}
|
||||||
return Report{Validated: true, Mutation: "activated", Release: release, ActiveRelease: updated.ActiveRelease, PreviousRelease: updated.PreviousRelease, EventWarnings: eventWarnings}, nil
|
report := Report{Validated: true, Mutation: "activated", Release: release, ActiveRelease: updated.ActiveRelease, PreviousRelease: updated.PreviousRelease, EventWarnings: eventWarnings}
|
||||||
|
if cfg.Deployment.Strategy == "singleton_candidate" {
|
||||||
|
if cleanupErr := state.RemoveCandidateLease(leasePath); cleanupErr != nil {
|
||||||
|
report.LeaseCleanupPending = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return report, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func singletonCandidateUnit(service, operationID string) (string, error) {
|
||||||
|
if len(operationID) != 32 {
|
||||||
|
return "", errors.New("candidate operation identity is invalid")
|
||||||
|
}
|
||||||
|
for _, character := range operationID {
|
||||||
|
if !strings.ContainsRune("0123456789abcdef", character) {
|
||||||
|
return "", errors.New("candidate operation identity is invalid")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return service + "-tend-candidate-" + operationID[:12] + ".service", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func clearCandidateLease(record *state.Record) {
|
||||||
|
record.CandidateRelease = ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadCandidateLease(cfg config.Config) (state.CandidateLease, bool, error) {
|
||||||
|
lease, err := state.LoadCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, cfg.Service.Name)
|
||||||
|
if err == nil {
|
||||||
|
return lease, true, nil
|
||||||
|
}
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return state.CandidateLease{}, false, nil
|
||||||
|
}
|
||||||
|
return state.CandidateLease{}, false, fmt.Errorf("load singleton candidate lease: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
type releaseIdentity struct {
|
type releaseIdentity struct {
|
||||||
@@ -194,13 +275,13 @@ func loadOrBootstrap(cfg config.Config, now time.Time) (state.Record, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return state.Record{}, fmt.Errorf("bootstrap active slot: %w", err)
|
return state.Record{}, fmt.Errorf("bootstrap active slot: %w", err)
|
||||||
}
|
}
|
||||||
return state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: slot, ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil
|
return state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: slot, ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil
|
||||||
case "singleton_candidate":
|
case "singleton_candidate":
|
||||||
release, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink)
|
release, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return state.Record{}, fmt.Errorf("bootstrap singleton: %w", err)
|
return state.Record{}, fmt.Errorf("bootstrap singleton: %w", err)
|
||||||
}
|
}
|
||||||
return state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil
|
return state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil
|
||||||
}
|
}
|
||||||
return state.Record{}, errors.New("unsupported strategy")
|
return state.Record{}, errors.New("unsupported strategy")
|
||||||
}
|
}
|
||||||
@@ -275,18 +356,18 @@ func (m Manager) deployBlueGreen(ctx context.Context, cfg config.Config, record
|
|||||||
if err = m.continuityWindow(ctx, cfg, previous.Address, true); err != nil {
|
if err = m.continuityWindow(ctx, cfg, previous.Address, true); err != nil {
|
||||||
return fmt.Errorf("activation continuity failed: %w", err)
|
return fmt.Errorf("activation continuity failed: %w", err)
|
||||||
}
|
}
|
||||||
next := state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: inactive, ActiveRelease: release, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, LastAttemptRelease: release, LastAttemptOutcome: "succeeded", LastAttemptAt: record.LastAttemptAt, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
next := state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: inactive, ActiveRelease: release, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, LastAttemptRelease: release, LastAttemptOutcome: "succeeded", LastAttemptAt: record.LastAttemptAt, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
||||||
if err = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next); err != nil {
|
if err = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m Manager) deploySingleton(ctx context.Context, cfg config.Config, record state.Record, release string) (err error) {
|
func (m Manager) deploySingleton(ctx context.Context, cfg config.Config, record state.Record, release, candidateUnit string) (err error) {
|
||||||
if err = m.activateSingletonRelease(ctx, cfg, release, true); err != nil {
|
if err = m.activateSingletonRelease(ctx, cfg, release, candidateUnit, true); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
next := state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, LastAttemptRelease: release, LastAttemptOutcome: "succeeded", LastAttemptAt: record.LastAttemptAt, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
next := state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, LastAttemptRelease: release, LastAttemptOutcome: "succeeded", LastAttemptAt: record.LastAttemptAt, UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
||||||
if err = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next); err != nil {
|
if err = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -306,12 +387,54 @@ func (m Manager) Rollback(ctx context.Context, cfg config.Config) (state.Record,
|
|||||||
if record.PreviousRelease == "" {
|
if record.PreviousRelease == "" {
|
||||||
return state.Record{}, errors.New("no previous release is recorded")
|
return state.Record{}, errors.New("no previous release is recorded")
|
||||||
}
|
}
|
||||||
|
leasePath := state.CandidateLeasePath(cfg.Deployment.StateFile)
|
||||||
|
if cfg.Deployment.Strategy == "singleton_candidate" {
|
||||||
|
_, exists, leaseErr := loadCandidateLease(cfg)
|
||||||
|
if leaseErr != nil {
|
||||||
|
return state.Record{}, leaseErr
|
||||||
|
}
|
||||||
|
if record.CandidateRelease != "" || exists {
|
||||||
|
return state.Record{}, errors.New("singleton candidate lease is unresolved; run tend reconcile --json before rollback")
|
||||||
|
}
|
||||||
|
}
|
||||||
started := m.Now()
|
started := m.Now()
|
||||||
identity, identityErr := m.ReadIdentity(record.PreviousRelease)
|
identity, identityErr := m.ReadIdentity(record.PreviousRelease)
|
||||||
digest, digestErr := releaseDigest(record.PreviousRelease)
|
digest, digestErr := releaseDigest(record.PreviousRelease)
|
||||||
operationID := ""
|
operationID := ""
|
||||||
if m.OperationID != nil {
|
if m.OperationID != nil {
|
||||||
operationID, _ = m.OperationID()
|
operationID, err = m.OperationID()
|
||||||
|
}
|
||||||
|
if cfg.Deployment.Strategy == "singleton_candidate" && (err != nil || operationID == "") {
|
||||||
|
return state.Record{}, errors.New("singleton rollback requires a fresh operation identity")
|
||||||
|
}
|
||||||
|
candidateUnit := ""
|
||||||
|
rollbackAttempt := record
|
||||||
|
if cfg.Deployment.Strategy == "singleton_candidate" {
|
||||||
|
candidateUnit, err = singletonCandidateUnit(cfg.Service.Name, operationID)
|
||||||
|
if err != nil {
|
||||||
|
return state.Record{}, err
|
||||||
|
}
|
||||||
|
attemptAt := m.Now().UTC().Format(time.RFC3339)
|
||||||
|
rollbackAttempt.DesiredRelease = record.PreviousRelease
|
||||||
|
rollbackAttempt.CandidateRelease = record.PreviousRelease
|
||||||
|
rollbackAttempt.LastAttemptRelease = record.PreviousRelease
|
||||||
|
rollbackAttempt.LastAttemptOutcome = "running"
|
||||||
|
rollbackAttempt.LastAttemptAt = attemptAt
|
||||||
|
rollbackAttempt.UpdatedAt = attemptAt
|
||||||
|
if err = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, rollbackAttempt); err != nil {
|
||||||
|
return state.Record{}, err
|
||||||
|
}
|
||||||
|
lease := state.CandidateLease{SchemaVersion: state.CandidateLeaseSchemaVersion, Service: cfg.Service.Name, OperationID: operationID, Release: record.PreviousRelease, Unit: candidateUnit, Address: cfg.Deployment.Singleton.CandidateAddress, StartedAt: attemptAt}
|
||||||
|
if err = state.StoreCandidateLease(leasePath, cfg.Deployment.Root, lease); err != nil {
|
||||||
|
failed := rollbackAttempt
|
||||||
|
clearCandidateLease(&failed)
|
||||||
|
failed.LastAttemptOutcome = "failed"
|
||||||
|
failed.UpdatedAt = m.Now().UTC().Format(time.RFC3339)
|
||||||
|
if storeErr := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, failed); storeErr != nil {
|
||||||
|
return state.Record{}, errors.Join(err, storeErr)
|
||||||
|
}
|
||||||
|
return state.Record{}, err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
emit := func(outcome string) {
|
emit := func(outcome string) {
|
||||||
if m.AppendEvent == nil || identityErr != nil || digestErr != nil || operationID == "" {
|
if m.AppendEvent == nil || identityErr != nil || digestErr != nil || operationID == "" {
|
||||||
@@ -325,16 +448,41 @@ func (m Manager) Rollback(ctx context.Context, cfg config.Config) (state.Record,
|
|||||||
case "blue_green":
|
case "blue_green":
|
||||||
err = m.rollbackBlueGreen(ctx, cfg, record)
|
err = m.rollbackBlueGreen(ctx, cfg, record)
|
||||||
case "singleton_candidate":
|
case "singleton_candidate":
|
||||||
err = m.rollbackSingleton(ctx, cfg, record)
|
err = m.rollbackSingleton(ctx, cfg, record, candidateUnit)
|
||||||
default:
|
default:
|
||||||
err = errors.New("unsupported strategy")
|
err = errors.New("unsupported strategy")
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if cfg.Deployment.Strategy == "singleton_candidate" {
|
||||||
|
failed := rollbackAttempt
|
||||||
|
var retained *retainedCandidateError
|
||||||
|
if !errors.As(err, &retained) {
|
||||||
|
if cleanupErr := state.RemoveCandidateLease(leasePath); cleanupErr != nil {
|
||||||
|
err = errors.Join(err, fmt.Errorf("candidate lease cleanup failed: %w", cleanupErr))
|
||||||
|
} else {
|
||||||
|
clearCandidateLease(&failed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
failed.LastAttemptOutcome = "failed"
|
||||||
|
failed.UpdatedAt = m.Now().UTC().Format(time.RFC3339)
|
||||||
|
if storeErr := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, failed); storeErr != nil {
|
||||||
|
err = errors.Join(err, storeErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
emit("failed")
|
emit("failed")
|
||||||
return state.Record{}, err
|
return state.Record{}, err
|
||||||
}
|
}
|
||||||
emit("succeeded")
|
emit("succeeded")
|
||||||
return state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
updated, loadErr := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||||
|
if loadErr != nil {
|
||||||
|
return state.Record{}, loadErr
|
||||||
|
}
|
||||||
|
if cfg.Deployment.Strategy == "singleton_candidate" {
|
||||||
|
if cleanupErr := state.RemoveCandidateLease(leasePath); cleanupErr != nil {
|
||||||
|
return updated, fmt.Errorf("rollback succeeded but candidate lease cleanup is pending; run tend reconcile --json: %w", cleanupErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return updated, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func releaseDigest(release string) (string, error) {
|
func releaseDigest(release string) (string, error) {
|
||||||
@@ -398,14 +546,14 @@ func (m Manager) rollbackBlueGreen(ctx context.Context, cfg config.Config, recor
|
|||||||
if err = m.probePublic(ctx, cfg, false); err != nil {
|
if err = m.probePublic(ctx, cfg, false); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
next := state.Record{SchemaVersion: 1, Strategy: record.Strategy, DesiredRelease: record.PreviousRelease, ActiveSlot: record.PreviousSlot, ActiveRelease: record.PreviousRelease, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, LastAttemptRelease: record.PreviousRelease, LastAttemptOutcome: "rolled_back", LastAttemptAt: m.Now().UTC().Format(time.RFC3339), UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
next := state.Record{SchemaVersion: state.SchemaVersion, Strategy: record.Strategy, DesiredRelease: record.PreviousRelease, ActiveSlot: record.PreviousSlot, ActiveRelease: record.PreviousRelease, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, LastAttemptRelease: record.PreviousRelease, LastAttemptOutcome: "rolled_back", LastAttemptAt: m.Now().UTC().Format(time.RFC3339), UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
||||||
return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next)
|
return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next)
|
||||||
}
|
}
|
||||||
func (m Manager) rollbackSingleton(ctx context.Context, cfg config.Config, record state.Record) (err error) {
|
func (m Manager) rollbackSingleton(ctx context.Context, cfg config.Config, record state.Record, candidateUnit string) (err error) {
|
||||||
if err = m.activateSingletonRelease(ctx, cfg, record.PreviousRelease, false); err != nil {
|
if err = m.activateSingletonRelease(ctx, cfg, record.PreviousRelease, candidateUnit, false); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
next := state.Record{SchemaVersion: 1, Strategy: record.Strategy, DesiredRelease: record.PreviousRelease, ActiveSlot: "singleton", ActiveRelease: record.PreviousRelease, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, LastAttemptRelease: record.PreviousRelease, LastAttemptOutcome: "rolled_back", LastAttemptAt: m.Now().UTC().Format(time.RFC3339), UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
next := state.Record{SchemaVersion: state.SchemaVersion, Strategy: record.Strategy, DesiredRelease: record.PreviousRelease, ActiveSlot: "singleton", ActiveRelease: record.PreviousRelease, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, LastAttemptRelease: record.PreviousRelease, LastAttemptOutcome: "rolled_back", LastAttemptAt: m.Now().UTC().Format(time.RFC3339), UpdatedAt: m.Now().UTC().Format(time.RFC3339)}
|
||||||
return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next)
|
return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -413,9 +561,8 @@ func (m Manager) rollbackSingleton(ctx context.Context, cfg config.Config, recor
|
|||||||
// installed fixed-address unit changes release. The transient candidate first
|
// installed fixed-address unit changes release. The transient candidate first
|
||||||
// receives traffic, remains healthy through the handoff, and is stopped only
|
// receives traffic, remains healthy through the handoff, and is stopped only
|
||||||
// after Caddy points back to the verified installed unit.
|
// after Caddy points back to the verified installed unit.
|
||||||
func (m Manager) activateSingletonRelease(ctx context.Context, cfg config.Config, release string, checkMarkers bool) (err error) {
|
func (m Manager) activateSingletonRelease(ctx context.Context, cfg config.Config, release, candidateUnit string, checkMarkers bool) (err error) {
|
||||||
single := *cfg.Deployment.Singleton
|
single := *cfg.Deployment.Singleton
|
||||||
candidateUnit := cfg.Service.Name + "-tend-candidate.service"
|
|
||||||
env := map[string]string{single.ListenEnv: single.CandidateAddress}
|
env := map[string]string{single.ListenEnv: single.CandidateAddress}
|
||||||
binary := filepath.Join(release, cfg.Build.Binary)
|
binary := filepath.Join(release, cfg.Build.Binary)
|
||||||
if err = m.Operator.StartCandidate(ctx, candidateUnit, binary, cfg.Service.EnvironmentFile, env); err != nil {
|
if err = m.Operator.StartCandidate(ctx, candidateUnit, binary, cfg.Service.EnvironmentFile, env); err != nil {
|
||||||
@@ -484,7 +631,7 @@ func (m Manager) activateSingletonRelease(ctx context.Context, cfg config.Config
|
|||||||
}
|
}
|
||||||
if recoveryErr != nil && handlerChanged {
|
if recoveryErr != nil && handlerChanged {
|
||||||
stopCandidate = false
|
stopCandidate = false
|
||||||
err = errors.Join(err, fmt.Errorf("singleton recovery incomplete; candidate remains routed for operator recovery: %w", recoveryErr))
|
err = &retainedCandidateError{cause: errors.Join(err, fmt.Errorf("singleton recovery incomplete; candidate remains routed for operator recovery: %w", recoveryErr))}
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
@@ -556,6 +703,15 @@ func (m Manager) Status(ctx context.Context, cfg config.Config) (Status, error)
|
|||||||
units = []string{cfg.Deployment.BlueGreen.Blue.Unit, cfg.Deployment.BlueGreen.Green.Unit}
|
units = []string{cfg.Deployment.BlueGreen.Blue.Unit, cfg.Deployment.BlueGreen.Green.Unit}
|
||||||
} else {
|
} else {
|
||||||
units = []string{cfg.Deployment.Singleton.Unit}
|
units = []string{cfg.Deployment.Singleton.Unit}
|
||||||
|
lease, exists, leaseErr := loadCandidateLease(cfg)
|
||||||
|
if leaseErr != nil {
|
||||||
|
return Status{}, leaseErr
|
||||||
|
}
|
||||||
|
if exists {
|
||||||
|
units = append(units, lease.Unit)
|
||||||
|
} else if result.StateInitialized && record.CandidateRelease != "" {
|
||||||
|
return Status{}, errors.New("candidate release has no operation-scoped lease; run tend reconcile --json")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
for _, unit := range units {
|
for _, unit := range units {
|
||||||
active, err := m.Operator.IsActive(ctx, unit)
|
active, err := m.Operator.IsActive(ctx, unit)
|
||||||
@@ -590,6 +746,17 @@ func (m Manager) Prune(cfg config.Config, keep int, apply bool) ([]string, error
|
|||||||
}
|
}
|
||||||
items := []candidate{}
|
items := []candidate{}
|
||||||
protected := map[string]bool{record.ActiveRelease: true, record.PreviousRelease: true}
|
protected := map[string]bool{record.ActiveRelease: true, record.PreviousRelease: true}
|
||||||
|
if cfg.Deployment.Strategy == "singleton_candidate" {
|
||||||
|
lease, exists, leaseErr := loadCandidateLease(cfg)
|
||||||
|
if leaseErr != nil {
|
||||||
|
return nil, leaseErr
|
||||||
|
}
|
||||||
|
if exists {
|
||||||
|
protected[lease.Release] = true
|
||||||
|
} else if record.CandidateRelease != "" {
|
||||||
|
protected[record.CandidateRelease] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
if !entry.IsDir() || entry.Type()&os.ModeSymlink != 0 || !strings.HasPrefix(entry.Name(), "sha256-") {
|
if !entry.IsDir() || entry.Type()&os.ModeSymlink != 0 || !strings.HasPrefix(entry.Name(), "sha256-") {
|
||||||
continue
|
continue
|
||||||
@@ -725,7 +892,7 @@ func resolveReleaseLink(root, link string) (string, error) {
|
|||||||
target = filepath.Join(filepath.Dir(link), target)
|
target = filepath.Join(filepath.Dir(link), target)
|
||||||
}
|
}
|
||||||
target = filepath.Clean(target)
|
target = filepath.Clean(target)
|
||||||
probe := state.Record{SchemaVersion: 1, Strategy: "singleton_candidate", ActiveSlot: "singleton", ActiveRelease: target, UpdatedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)}
|
probe := state.Record{SchemaVersion: state.SchemaVersion, Strategy: "singleton_candidate", ActiveSlot: "singleton", ActiveRelease: target, UpdatedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)}
|
||||||
if err := probe.Validate(root, "singleton_candidate"); err != nil {
|
if err := probe.Validate(root, "singleton_candidate"); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
|||||||
+141
-25
@@ -17,6 +17,16 @@ import (
|
|||||||
"gamertan.com/tend/internal/state"
|
"gamertan.com/tend/internal/state"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const testCandidateUnit = "example-site-tend-candidate-dddddddddddd.service"
|
||||||
|
|
||||||
|
func storeTestCandidateLease(t *testing.T, cfg config.Config, release, startedAt string) {
|
||||||
|
t.Helper()
|
||||||
|
lease := state.CandidateLease{SchemaVersion: state.CandidateLeaseSchemaVersion, Service: cfg.Service.Name, OperationID: strings.Repeat("d", 32), Release: release, Unit: testCandidateUnit, Address: cfg.Deployment.Singleton.CandidateAddress, StartedAt: startedAt}
|
||||||
|
if err := state.StoreCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, lease); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type fakeOperator struct {
|
type fakeOperator struct {
|
||||||
failReload bool
|
failReload bool
|
||||||
failReloadAt int
|
failReloadAt int
|
||||||
@@ -296,30 +306,48 @@ func TestDeploymentEvidenceCanNeverBlockActivationOrRollback(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestDeploymentEvidenceIdentityAndEntropyAreBestEffort(t *testing.T) {
|
func TestDeploymentEvidenceIdentityIsBestEffort(t *testing.T) {
|
||||||
for _, test := range []struct {
|
|
||||||
name string
|
|
||||||
damage func(*Manager)
|
|
||||||
}{
|
|
||||||
{"identity", func(manager *Manager) {
|
|
||||||
manager.ReadIdentity = func(string) (releaseIdentity, error) {
|
|
||||||
return releaseIdentity{}, errors.New("injected identity failure")
|
|
||||||
}
|
|
||||||
}},
|
|
||||||
{"entropy", func(manager *Manager) {
|
|
||||||
manager.OperationID = func() (string, error) { return "", errors.New("injected entropy failure") }
|
|
||||||
}},
|
|
||||||
} {
|
|
||||||
t.Run(test.name, func(t *testing.T) {
|
|
||||||
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||||
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
|
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
|
||||||
manager := manager(&fakeOperator{active: map[string]bool{}}, fresh)
|
m := manager(&fakeOperator{active: map[string]bool{}}, fresh)
|
||||||
test.damage(&manager)
|
m.ReadIdentity = func(string) (releaseIdentity, error) {
|
||||||
report, err := manager.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
|
return releaseIdentity{}, errors.New("injected identity failure")
|
||||||
|
}
|
||||||
|
report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)})
|
||||||
if err != nil || report.EventWarnings != 1 || report.ActiveRelease != fresh {
|
if err != nil || report.EventWarnings != 1 || report.ActiveRelease != fresh {
|
||||||
t.Fatalf("report=%+v err=%v", report, err)
|
t.Fatalf("report=%+v err=%v", report, err)
|
||||||
}
|
}
|
||||||
})
|
}
|
||||||
|
|
||||||
|
func TestSingletonOperationIdentityIsRequiredBeforeCandidateStart(t *testing.T) {
|
||||||
|
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||||
|
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
|
||||||
|
operator := &fakeOperator{active: map[string]bool{}}
|
||||||
|
m := manager(operator, fresh)
|
||||||
|
m.OperationID = func() (string, error) { return "", errors.New("injected entropy failure") }
|
||||||
|
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err == nil || !strings.Contains(err.Error(), "operation identity") {
|
||||||
|
t.Fatalf("expected operation identity refusal, got %v", err)
|
||||||
|
}
|
||||||
|
if len(operator.starts) != 0 {
|
||||||
|
t.Fatalf("candidate started without an operation identity: %#v", operator.starts)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSingletonUnresolvedLeaseBlocksReplacementBeforeCandidateStart(t *testing.T) {
|
||||||
|
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||||
|
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
|
||||||
|
at := time.Unix(100, 0).UTC().Format(time.RFC3339)
|
||||||
|
record := state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: fresh, CandidateRelease: fresh, ActiveSlot: "singleton", ActiveRelease: old, LastAttemptRelease: fresh, LastAttemptOutcome: "failed", LastAttemptAt: at, UpdatedAt: at}
|
||||||
|
if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
storeTestCandidateLease(t, cfg, fresh, at)
|
||||||
|
operator := &fakeOperator{active: map[string]bool{testCandidateUnit: true}}
|
||||||
|
if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err == nil || !strings.Contains(err.Error(), "run tend reconcile") {
|
||||||
|
t.Fatalf("expected unresolved lease refusal, got %v", err)
|
||||||
|
}
|
||||||
|
if len(operator.starts) != 0 || !operator.active[testCandidateUnit] {
|
||||||
|
t.Fatalf("existing candidate was disturbed: starts=%#v active=%#v", operator.starts, operator.active)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -380,7 +408,7 @@ func TestSingletonRestartFailureRestoresPointers(t *testing.T) {
|
|||||||
if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 {
|
if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 {
|
||||||
t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err)
|
t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err)
|
||||||
}
|
}
|
||||||
if operator.active["example-site-tend-candidate.service"] {
|
if operator.active[testCandidateUnit] {
|
||||||
t.Fatal("candidate was not stopped after successful restoration")
|
t.Fatal("candidate was not stopped after successful restoration")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -398,9 +426,15 @@ func TestStateRecordsSuccessfulActivation(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if record.ActiveRelease != fresh || record.PreviousRelease != old {
|
if record.ActiveRelease != fresh || record.PreviousRelease != old || record.CandidateRelease != "" {
|
||||||
t.Fatalf("state=%+v", record)
|
t.Fatalf("state=%+v", record)
|
||||||
}
|
}
|
||||||
|
if _, err := os.Lstat(state.CandidateLeasePath(cfg.Deployment.StateFile)); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("candidate lease was not removed: %v", err)
|
||||||
|
}
|
||||||
|
if len(operator.starts) != 1 || operator.starts[0] != testCandidateUnit {
|
||||||
|
t.Fatalf("candidate starts=%#v", operator.starts)
|
||||||
|
}
|
||||||
if operator.candidateFile != cfg.Service.EnvironmentFile || len(operator.candidateEnvironment) != 1 || operator.candidateEnvironment["EXAMPLE_LISTEN"] != "127.0.0.1:18092" {
|
if operator.candidateFile != cfg.Service.EnvironmentFile || len(operator.candidateEnvironment) != 1 || operator.candidateEnvironment["EXAMPLE_LISTEN"] != "127.0.0.1:18092" {
|
||||||
t.Fatalf("candidate file=%q environment=%#v", operator.candidateFile, operator.candidateEnvironment)
|
t.Fatalf("candidate file=%q environment=%#v", operator.candidateFile, operator.candidateEnvironment)
|
||||||
}
|
}
|
||||||
@@ -408,9 +442,13 @@ func TestStateRecordsSuccessfulActivation(t *testing.T) {
|
|||||||
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
|
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
|
||||||
t.Fatalf("handler=%q err=%v", body, err)
|
t.Fatalf("handler=%q err=%v", body, err)
|
||||||
}
|
}
|
||||||
if operator.reloads != 2 || operator.active["example-site-tend-candidate.service"] {
|
if operator.reloads != 2 || operator.active[testCandidateUnit] {
|
||||||
t.Fatalf("reloads=%d active=%#v", operator.reloads, operator.active)
|
t.Fatalf("reloads=%d active=%#v", operator.reloads, operator.active)
|
||||||
}
|
}
|
||||||
|
reconciliation, err := m.Reconcile(context.Background(), cfg)
|
||||||
|
if err != nil || reconciliation.Mutation != "none" || reconciliation.Disposition != "settled" || reconciliation.Observed.CandidateLease || !reconciliation.Consistent {
|
||||||
|
t.Fatalf("reconciliation=%+v err=%v", reconciliation, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSingletonContinuityFailureRestoresHandlerPointersAndService(t *testing.T) {
|
func TestSingletonContinuityFailureRestoresHandlerPointersAndService(t *testing.T) {
|
||||||
@@ -430,7 +468,7 @@ func TestSingletonContinuityFailureRestoresHandlerPointersAndService(t *testing.
|
|||||||
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
|
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
|
||||||
t.Fatalf("handler=%q err=%v", body, err)
|
t.Fatalf("handler=%q err=%v", body, err)
|
||||||
}
|
}
|
||||||
if operator.active["example-site-tend-candidate.service"] {
|
if operator.active[testCandidateUnit] {
|
||||||
t.Fatal("candidate was not stopped after continuity restoration")
|
t.Fatal("candidate was not stopped after continuity restoration")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -453,7 +491,7 @@ func TestSingletonCaddyReloadFailuresRestorePriorRoute(t *testing.T) {
|
|||||||
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
|
if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" {
|
||||||
t.Fatalf("handler=%q err=%v", body, err)
|
t.Fatalf("handler=%q err=%v", body, err)
|
||||||
}
|
}
|
||||||
if operator.active["example-site-tend-candidate.service"] {
|
if operator.active[testCandidateUnit] {
|
||||||
t.Fatal("candidate was not stopped after route restoration")
|
t.Fatal("candidate was not stopped after route restoration")
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -469,7 +507,85 @@ func TestSingletonIncompleteRecoveryKeepsProvenCandidateRunning(t *testing.T) {
|
|||||||
if err == nil || !strings.Contains(err.Error(), "candidate remains routed for operator recovery") {
|
if err == nil || !strings.Contains(err.Error(), "candidate remains routed for operator recovery") {
|
||||||
t.Fatalf("expected explicit incomplete recovery, got %v", err)
|
t.Fatalf("expected explicit incomplete recovery, got %v", err)
|
||||||
}
|
}
|
||||||
if !operator.active["example-site-tend-candidate.service"] {
|
if !operator.active[testCandidateUnit] {
|
||||||
t.Fatal("proven candidate was stopped despite incomplete route restoration")
|
t.Fatal("proven candidate was stopped despite incomplete route restoration")
|
||||||
}
|
}
|
||||||
|
record, loadErr := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||||
|
lease, leaseErr := state.LoadCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, cfg.Service.Name)
|
||||||
|
if loadErr != nil || leaseErr != nil || record.CandidateRelease != fresh || lease.OperationID != strings.Repeat("d", 32) || lease.Unit != testCandidateUnit {
|
||||||
|
t.Fatalf("retained state=%+v err=%v", record, loadErr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReconcileReportsRetainedCandidateWithoutMutation(t *testing.T) {
|
||||||
|
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||||
|
settings, handler := singletonSettings(t, cfg, old)
|
||||||
|
cfg.Deployment.Singleton = settings
|
||||||
|
at := time.Unix(100, 0).UTC().Format(time.RFC3339)
|
||||||
|
record := state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: fresh, CandidateRelease: fresh, ActiveSlot: "singleton", ActiveRelease: old, LastAttemptRelease: fresh, LastAttemptOutcome: "failed", LastAttemptAt: at, UpdatedAt: at}
|
||||||
|
if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
storeTestCandidateLease(t, cfg, fresh, at)
|
||||||
|
candidateHandler, err := renderHandler(settings.CaddyHandlerTemplate, settings.CandidateAddress)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(handler, candidateHandler, 0o640); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
operator := &fakeOperator{active: map[string]bool{settings.Unit: true, testCandidateUnit: true}}
|
||||||
|
reconciliation, err := manager(operator, fresh).Reconcile(context.Background(), cfg)
|
||||||
|
if err != nil || reconciliation.Mutation != "none" || !reconciliation.Observed.CandidateLease || reconciliation.Observed.CandidateUnitActive == nil || !*reconciliation.Observed.CandidateUnitActive || reconciliation.Observed.HandlerFileTarget != "candidate" || reconciliation.Disposition != "retained_candidate_handler_file" {
|
||||||
|
t.Fatalf("reconciliation=%+v err=%v", reconciliation, err)
|
||||||
|
}
|
||||||
|
if len(operator.stops) != 0 || len(operator.restarts) != 0 || operator.reloads != 0 {
|
||||||
|
t.Fatalf("reconcile mutated services: stops=%#v restarts=%#v reloads=%d", operator.stops, operator.restarts, operator.reloads)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSingletonRollbackRetainsOperationLeaseWhenRecoveryIsIncomplete(t *testing.T) {
|
||||||
|
cfg, old, fresh := baseConfig(t, "singleton_candidate")
|
||||||
|
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old)
|
||||||
|
operator := &fakeOperator{active: map[string]bool{cfg.Deployment.Singleton.Unit: true}}
|
||||||
|
m := manager(operator, fresh)
|
||||||
|
if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
operator.failReload = true
|
||||||
|
if _, err := m.Rollback(context.Background(), cfg); err == nil || !strings.Contains(err.Error(), "candidate remains routed for operator recovery") {
|
||||||
|
t.Fatalf("expected retained rollback candidate, got %v", err)
|
||||||
|
}
|
||||||
|
record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
lease, leaseErr := state.LoadCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, cfg.Service.Name)
|
||||||
|
if record.ActiveRelease != fresh || record.CandidateRelease != old || leaseErr != nil || lease.OperationID != strings.Repeat("d", 32) || lease.Unit != testCandidateUnit || record.LastAttemptOutcome != "failed" {
|
||||||
|
t.Fatalf("rollback state=%+v", record)
|
||||||
|
}
|
||||||
|
if !operator.active[testCandidateUnit] {
|
||||||
|
t.Fatal("rollback candidate was stopped despite incomplete recovery")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPruneProtectsOperationScopedCandidateRelease(t *testing.T) {
|
||||||
|
cfg, active, candidate := baseConfig(t, "singleton_candidate")
|
||||||
|
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, active)
|
||||||
|
at := time.Unix(100, 0).UTC().Format(time.RFC3339)
|
||||||
|
record := state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: candidate, CandidateRelease: candidate, ActiveSlot: "singleton", ActiveRelease: active, LastAttemptRelease: candidate, LastAttemptOutcome: "failed", LastAttemptAt: at, UpdatedAt: at}
|
||||||
|
if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
storeTestCandidateLease(t, cfg, candidate, at)
|
||||||
|
removed, err := manager(&fakeOperator{active: map[string]bool{}}, candidate).Prune(cfg, 2, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(removed) != 0 {
|
||||||
|
t.Fatalf("candidate release was selected for pruning: %#v", removed)
|
||||||
|
}
|
||||||
|
if info, err := os.Stat(candidate); err != nil || !info.IsDir() {
|
||||||
|
t.Fatalf("candidate release was not preserved: %v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,270 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
package deploy
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"gamertan.com/tend/internal/config"
|
||||||
|
"gamertan.com/tend/internal/state"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Finding struct {
|
||||||
|
Code string `json:"code"`
|
||||||
|
Severity string `json:"severity"`
|
||||||
|
Message string `json:"message"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ObservedReleaseIdentity struct {
|
||||||
|
Version string `json:"version"`
|
||||||
|
Commit string `json:"commit"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ObservedState struct {
|
||||||
|
ActiveRelease string `json:"active_release,omitempty"`
|
||||||
|
PreviousRelease string `json:"previous_release,omitempty"`
|
||||||
|
ActiveIdentity *ObservedReleaseIdentity `json:"active_identity,omitempty"`
|
||||||
|
Units map[string]bool `json:"units"`
|
||||||
|
CandidateLease bool `json:"candidate_lease"`
|
||||||
|
CandidateUnit string `json:"candidate_unit,omitempty"`
|
||||||
|
CandidateUnitActive *bool `json:"candidate_unit_active,omitempty"`
|
||||||
|
LegacyCandidateUnit string `json:"legacy_candidate_unit,omitempty"`
|
||||||
|
LegacyCandidateActive *bool `json:"legacy_candidate_unit_active,omitempty"`
|
||||||
|
RouteHandlerMatches bool `json:"route_handler_matches"`
|
||||||
|
HandlerFileTarget string `json:"handler_file_target,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Reconciliation struct {
|
||||||
|
Service string `json:"service"`
|
||||||
|
Strategy string `json:"strategy"`
|
||||||
|
Mutation string `json:"mutation"`
|
||||||
|
Consistent bool `json:"consistent"`
|
||||||
|
StateInitialized bool `json:"state_initialized"`
|
||||||
|
State *state.Record `json:"state,omitempty"`
|
||||||
|
Observed ObservedState `json:"observed"`
|
||||||
|
Disposition string `json:"disposition"`
|
||||||
|
Findings []Finding `json:"findings"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reconcile observes configured units, release pointers, installed release
|
||||||
|
// identity, and the imported Caddy handler. It never acquires the deployment
|
||||||
|
// lock or mutates service state; proposed repairs remain an operator decision.
|
||||||
|
func (m Manager) Reconcile(ctx context.Context, cfg config.Config) (Reconciliation, error) {
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
return Reconciliation{}, err
|
||||||
|
}
|
||||||
|
if m.Operator == nil || m.ReadIdentity == nil {
|
||||||
|
return Reconciliation{}, errors.New("reconciliation dependencies are unavailable")
|
||||||
|
}
|
||||||
|
report := Reconciliation{
|
||||||
|
Service: cfg.Service.Name, Strategy: cfg.Deployment.Strategy, Mutation: "none",
|
||||||
|
Observed: ObservedState{Units: map[string]bool{}}, Findings: []Finding{},
|
||||||
|
}
|
||||||
|
add := func(code, severity, message string) {
|
||||||
|
report.Findings = append(report.Findings, Finding{Code: code, Severity: severity, Message: message})
|
||||||
|
}
|
||||||
|
record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy)
|
||||||
|
if err == nil {
|
||||||
|
report.State = &record
|
||||||
|
report.StateInitialized = true
|
||||||
|
} else if os.IsNotExist(err) {
|
||||||
|
add("state_uninitialized", "warning", "Tend has no validated state record for this service.")
|
||||||
|
} else {
|
||||||
|
add("state_invalid", "error", "The Tend state record could not be validated.")
|
||||||
|
}
|
||||||
|
|
||||||
|
activeSlot := ""
|
||||||
|
activeUnit := ""
|
||||||
|
activeAddress := ""
|
||||||
|
handler := ""
|
||||||
|
template := ""
|
||||||
|
var candidateLease *state.CandidateLease
|
||||||
|
switch cfg.Deployment.Strategy {
|
||||||
|
case "singleton_candidate":
|
||||||
|
single := *cfg.Deployment.Singleton
|
||||||
|
activeSlot = "singleton"
|
||||||
|
activeUnit = single.Unit
|
||||||
|
activeAddress = single.Address
|
||||||
|
handler, template = single.CaddyHandler, single.CaddyHandlerTemplate
|
||||||
|
report.Observed.ActiveRelease = observeReleaseLink(cfg, single.CurrentLink, true, add)
|
||||||
|
report.Observed.PreviousRelease = observeReleaseLink(cfg, single.PreviousLink, false, add)
|
||||||
|
legacyCandidateUnit := cfg.Service.Name + "-tend-candidate.service"
|
||||||
|
candidateUnit := legacyCandidateUnit
|
||||||
|
lease, leaseErr := state.LoadCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, cfg.Service.Name)
|
||||||
|
if leaseErr == nil {
|
||||||
|
candidateLease = &lease
|
||||||
|
report.Observed.CandidateLease = true
|
||||||
|
candidateUnit = lease.Unit
|
||||||
|
} else if !os.IsNotExist(leaseErr) {
|
||||||
|
report.Observed.CandidateLease = true
|
||||||
|
candidateUnit = ""
|
||||||
|
add("candidate_lease_invalid", "error", "The operation-scoped candidate lease could not be validated.")
|
||||||
|
} else if report.State != nil && report.State.CandidateRelease != "" {
|
||||||
|
report.Observed.CandidateLease = true
|
||||||
|
candidateUnit = ""
|
||||||
|
add("legacy_candidate_lease", "error", "The state records a candidate release without an operation-scoped lease and requires manual review.")
|
||||||
|
}
|
||||||
|
if candidateUnit != "" {
|
||||||
|
report.Observed.CandidateUnit = candidateUnit
|
||||||
|
candidateActive, candidateErr := m.Operator.IsActive(ctx, candidateUnit)
|
||||||
|
if candidateErr != nil {
|
||||||
|
add("candidate_unit_unobservable", "error", "The transient candidate unit state could not be observed.")
|
||||||
|
} else {
|
||||||
|
report.Observed.CandidateUnitActive = &candidateActive
|
||||||
|
report.Observed.Units[candidateUnit] = candidateActive
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if candidateUnit != legacyCandidateUnit {
|
||||||
|
report.Observed.LegacyCandidateUnit = legacyCandidateUnit
|
||||||
|
legacyActive, legacyErr := m.Operator.IsActive(ctx, legacyCandidateUnit)
|
||||||
|
if legacyErr != nil {
|
||||||
|
add("legacy_candidate_unit_unobservable", "error", "The legacy fixed candidate unit state could not be observed.")
|
||||||
|
} else {
|
||||||
|
report.Observed.LegacyCandidateActive = &legacyActive
|
||||||
|
report.Observed.Units[legacyCandidateUnit] = legacyActive
|
||||||
|
if legacyActive {
|
||||||
|
add("legacy_candidate_unit_active", "error", "A legacy fixed-name candidate remains active beside an operation-scoped lease.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "blue_green":
|
||||||
|
blueGreen := *cfg.Deployment.BlueGreen
|
||||||
|
handler, template = blueGreen.CaddyHandler, blueGreen.CaddyHandlerTemplate
|
||||||
|
activeSlot = blueGreen.BootstrapActive
|
||||||
|
if report.State != nil {
|
||||||
|
activeSlot = report.State.ActiveSlot
|
||||||
|
}
|
||||||
|
active := slotConfig(blueGreen, activeSlot)
|
||||||
|
previousName := "blue"
|
||||||
|
if activeSlot == "blue" {
|
||||||
|
previousName = "green"
|
||||||
|
}
|
||||||
|
previous := slotConfig(blueGreen, previousName)
|
||||||
|
activeUnit, activeAddress = active.Unit, active.Address
|
||||||
|
report.Observed.ActiveRelease = observeReleaseLink(cfg, active.Link, true, add)
|
||||||
|
report.Observed.PreviousRelease = observeReleaseLink(cfg, previous.Link, false, add)
|
||||||
|
for _, slot := range []config.Slot{blueGreen.Blue, blueGreen.Green} {
|
||||||
|
observeUnit(ctx, m.Operator, slot.Unit, report.Observed.Units, add)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, exists := report.Observed.Units[activeUnit]; !exists {
|
||||||
|
observeUnit(ctx, m.Operator, activeUnit, report.Observed.Units, add)
|
||||||
|
}
|
||||||
|
if active, observed := report.Observed.Units[activeUnit]; activeUnit != "" && observed && !active {
|
||||||
|
add("active_unit_inactive", "error", "The configured active service unit is not active.")
|
||||||
|
}
|
||||||
|
|
||||||
|
if report.Observed.ActiveRelease != "" {
|
||||||
|
identity, identityErr := m.ReadIdentity(report.Observed.ActiveRelease)
|
||||||
|
if identityErr != nil {
|
||||||
|
add("active_identity_unreadable", "error", "The observed active release identity could not be validated.")
|
||||||
|
} else {
|
||||||
|
report.Observed.ActiveIdentity = &ObservedReleaseIdentity{Version: identity.Version, Commit: identity.Commit}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
expectedHandler, renderErr := renderHandler(template, activeAddress)
|
||||||
|
actualHandler, readErr := os.ReadFile(handler)
|
||||||
|
if renderErr != nil || readErr != nil {
|
||||||
|
add("route_handler_unreadable", "error", "The configured Caddy handler or its template could not be validated.")
|
||||||
|
} else {
|
||||||
|
report.Observed.RouteHandlerMatches = bytes.Equal(expectedHandler, actualHandler)
|
||||||
|
if report.Observed.RouteHandlerMatches {
|
||||||
|
report.Observed.HandlerFileTarget = "installed"
|
||||||
|
} else if cfg.Deployment.Strategy == "singleton_candidate" {
|
||||||
|
candidateAddress := cfg.Deployment.Singleton.CandidateAddress
|
||||||
|
if candidateLease != nil {
|
||||||
|
candidateAddress = candidateLease.Address
|
||||||
|
}
|
||||||
|
candidateHandler, candidateErr := renderHandler(template, candidateAddress)
|
||||||
|
if candidateErr == nil && bytes.Equal(candidateHandler, actualHandler) {
|
||||||
|
report.Observed.HandlerFileTarget = "candidate"
|
||||||
|
} else {
|
||||||
|
report.Observed.HandlerFileTarget = "other"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !report.Observed.RouteHandlerMatches {
|
||||||
|
add("route_handler_drift", "error", "The installed Caddy handler does not match the configured active upstream.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if report.State != nil {
|
||||||
|
if report.State.ActiveSlot != activeSlot || report.State.ActiveRelease != report.Observed.ActiveRelease {
|
||||||
|
add("active_release_drift", "error", "Recorded active state does not match the observed active release pointer.")
|
||||||
|
}
|
||||||
|
if report.State.PreviousRelease != report.Observed.PreviousRelease {
|
||||||
|
add("previous_release_drift", "warning", "Recorded rollback state does not match the observed previous release pointer.")
|
||||||
|
}
|
||||||
|
if cfg.Deployment.Strategy == "singleton_candidate" {
|
||||||
|
leased := report.Observed.CandidateLease && candidateLease != nil
|
||||||
|
candidateActive := report.Observed.CandidateUnitActive != nil && *report.Observed.CandidateUnitActive
|
||||||
|
if candidateLease != nil && report.State.CandidateRelease == "" {
|
||||||
|
add("candidate_lease_without_running_attempt", "error", "An operation-scoped candidate lease remains after the recorded attempt settled.")
|
||||||
|
}
|
||||||
|
if candidateLease != nil && report.State.CandidateRelease != "" && candidateLease.Release != report.State.CandidateRelease {
|
||||||
|
add("candidate_lease_release_mismatch", "error", "The operation-scoped candidate lease does not match the recorded candidate release.")
|
||||||
|
}
|
||||||
|
if leased && report.Observed.CandidateUnitActive != nil && !candidateActive {
|
||||||
|
add("inactive_candidate_lease", "error", "State retains a candidate lease but its operation-scoped unit is inactive.")
|
||||||
|
}
|
||||||
|
if !report.Observed.CandidateLease && candidateActive {
|
||||||
|
add("unleased_candidate_active", "error", "A transient candidate unit is active without a matching running attempt.")
|
||||||
|
}
|
||||||
|
if leased && candidateActive && report.Observed.HandlerFileTarget == "candidate" {
|
||||||
|
add("retained_candidate_routed", "warning", "The retained candidate appears in the handler file; do not stop it before establishing another healthy route.")
|
||||||
|
}
|
||||||
|
if leased && candidateActive && report.Observed.HandlerFileTarget != "candidate" {
|
||||||
|
add("candidate_active_not_routed", "warning", "The leased candidate is active but the handler file does not target it; cleanup remains an explicit reviewed operation.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case !report.StateInitialized:
|
||||||
|
report.Disposition = "state_uninitialized"
|
||||||
|
case report.Observed.CandidateLease && report.Observed.CandidateUnit == "":
|
||||||
|
report.Disposition = "legacy_or_invalid_candidate_lease"
|
||||||
|
case report.Observed.CandidateLease && report.Observed.CandidateUnitActive != nil && *report.Observed.CandidateUnitActive && report.Observed.HandlerFileTarget == "candidate":
|
||||||
|
report.Disposition = "retained_candidate_handler_file"
|
||||||
|
case report.Observed.CandidateLease && report.Observed.CandidateUnitActive != nil && *report.Observed.CandidateUnitActive:
|
||||||
|
report.Disposition = "candidate_active_not_in_handler_file"
|
||||||
|
case report.Observed.CandidateLease:
|
||||||
|
report.Disposition = "inactive_candidate_lease"
|
||||||
|
case len(report.Findings) == 0:
|
||||||
|
report.Disposition = "settled"
|
||||||
|
default:
|
||||||
|
report.Disposition = "manual_review_required"
|
||||||
|
}
|
||||||
|
report.Consistent = len(report.Findings) == 0
|
||||||
|
return report, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func observeReleaseLink(cfg config.Config, link string, required bool, add func(string, string, string)) string {
|
||||||
|
release, err := resolveReleaseLink(cfg.Deployment.Root, link)
|
||||||
|
if err == nil {
|
||||||
|
return release
|
||||||
|
}
|
||||||
|
if !required && os.IsNotExist(err) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
code := "previous_pointer_unreadable"
|
||||||
|
message := "The configured previous release pointer could not be validated."
|
||||||
|
severity := "warning"
|
||||||
|
if required {
|
||||||
|
code = "active_pointer_unreadable"
|
||||||
|
message = "The configured active release pointer could not be validated."
|
||||||
|
severity = "error"
|
||||||
|
}
|
||||||
|
add(code, severity, message)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func observeUnit(ctx context.Context, operator Operator, unit string, units map[string]bool, add func(string, string, string)) {
|
||||||
|
active, err := operator.IsActive(ctx, unit)
|
||||||
|
if err != nil {
|
||||||
|
add("unit_unobservable", "error", "A configured service unit state could not be observed.")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
units[unit] = active
|
||||||
|
}
|
||||||
@@ -0,0 +1,182 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
package deploy
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gamertan.com/tend/internal/config"
|
||||||
|
"gamertan.com/tend/internal/state"
|
||||||
|
)
|
||||||
|
|
||||||
|
func writeReleaseIdentity(t *testing.T, release, version, commit string) {
|
||||||
|
t.Helper()
|
||||||
|
body := `{"version":"` + version + `","commit":"` + commit + `"}`
|
||||||
|
if err := os.WriteFile(filepath.Join(release, "RELEASE.json"), []byte(body), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func storeSingletonState(t *testing.T, cfg config.Config, active, previous string) {
|
||||||
|
t.Helper()
|
||||||
|
record := state.Record{
|
||||||
|
SchemaVersion: state.SchemaVersion,
|
||||||
|
Strategy: "singleton_candidate",
|
||||||
|
DesiredRelease: active,
|
||||||
|
ActiveSlot: "singleton",
|
||||||
|
ActiveRelease: active,
|
||||||
|
PreviousRelease: previous,
|
||||||
|
LastAttemptRelease: active,
|
||||||
|
LastAttemptOutcome: "succeeded",
|
||||||
|
LastAttemptAt: time.Unix(100, 0).UTC().Format(time.RFC3339),
|
||||||
|
UpdatedAt: time.Unix(100, 0).UTC().Format(time.RFC3339),
|
||||||
|
}
|
||||||
|
if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func reconciliationManager(operator Operator) Manager {
|
||||||
|
return Manager{Operator: operator, ReadIdentity: readReleaseIdentity}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReconcileReportsHealthySingletonWithoutMutation(t *testing.T) {
|
||||||
|
cfg, active, previous := baseConfig(t, "singleton_candidate")
|
||||||
|
cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, active)
|
||||||
|
if err := replaceSymlink(cfg.Deployment.Singleton.PreviousLink, previous); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
commit := strings.Repeat("a", 40)
|
||||||
|
writeReleaseIdentity(t, active, "v0.2.0-preview.2", commit)
|
||||||
|
storeSingletonState(t, cfg, active, previous)
|
||||||
|
operator := &fakeOperator{active: map[string]bool{
|
||||||
|
cfg.Deployment.Singleton.Unit: true,
|
||||||
|
cfg.Service.Name + "-tend-candidate.service": false,
|
||||||
|
}}
|
||||||
|
|
||||||
|
report, err := reconciliationManager(operator).Reconcile(context.Background(), cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !report.Consistent || report.Mutation != "none" || !report.StateInitialized || len(report.Findings) != 0 {
|
||||||
|
t.Fatalf("report=%+v", report)
|
||||||
|
}
|
||||||
|
if report.Observed.ActiveRelease != active || report.Observed.PreviousRelease != previous || !report.Observed.RouteHandlerMatches {
|
||||||
|
t.Fatalf("observed=%+v", report.Observed)
|
||||||
|
}
|
||||||
|
if report.Observed.ActiveIdentity == nil || report.Observed.ActiveIdentity.Version != "v0.2.0-preview.2" || report.Observed.ActiveIdentity.Commit != commit {
|
||||||
|
t.Fatalf("identity=%+v", report.Observed.ActiveIdentity)
|
||||||
|
}
|
||||||
|
if !report.Observed.Units[cfg.Deployment.Singleton.Unit] || report.Observed.CandidateUnitActive == nil || *report.Observed.CandidateUnitActive {
|
||||||
|
t.Fatalf("units=%#v candidate=%v", report.Observed.Units, report.Observed.CandidateUnitActive)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReconcileReportsHealthyBlueGreenDeployment(t *testing.T) {
|
||||||
|
cfg, active, previous := baseConfig(t, "blue_green")
|
||||||
|
handler := filepath.Join(cfg.Deployment.Root, "handler.caddy")
|
||||||
|
template := filepath.Join(cfg.Deployment.Root, "handler.template")
|
||||||
|
if err := os.WriteFile(handler, []byte("reverse_proxy 127.0.0.1:8090\n"), 0o640); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
blue := filepath.Join(cfg.Deployment.Root, "slots", "blue")
|
||||||
|
green := filepath.Join(cfg.Deployment.Root, "slots", "green")
|
||||||
|
if err := replaceSymlink(blue, active); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := replaceSymlink(green, previous); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cfg.Deployment.BlueGreen = &config.BlueGreen{
|
||||||
|
CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler,
|
||||||
|
CaddyHandlerTemplate: template, BootstrapActive: "blue",
|
||||||
|
Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue},
|
||||||
|
Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green},
|
||||||
|
}
|
||||||
|
writeReleaseIdentity(t, active, "v0.2.0-preview.2", strings.Repeat("c", 40))
|
||||||
|
record := state.Record{
|
||||||
|
SchemaVersion: state.SchemaVersion, Strategy: "blue_green", DesiredRelease: active,
|
||||||
|
ActiveSlot: "blue", ActiveRelease: active, PreviousSlot: "green", PreviousRelease: previous,
|
||||||
|
LastAttemptRelease: active, LastAttemptOutcome: "succeeded",
|
||||||
|
LastAttemptAt: time.Unix(100, 0).UTC().Format(time.RFC3339), UpdatedAt: time.Unix(100, 0).UTC().Format(time.RFC3339),
|
||||||
|
}
|
||||||
|
if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
operator := &fakeOperator{active: map[string]bool{
|
||||||
|
"example-blue.service": true, "example-green.service": true,
|
||||||
|
}}
|
||||||
|
|
||||||
|
report, err := reconciliationManager(operator).Reconcile(context.Background(), cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !report.Consistent || report.Mutation != "none" || report.Observed.ActiveRelease != active || report.Observed.PreviousRelease != previous || !report.Observed.RouteHandlerMatches {
|
||||||
|
t.Fatalf("report=%+v", report)
|
||||||
|
}
|
||||||
|
if report.Observed.CandidateUnit != "" || report.Observed.CandidateUnitActive != nil {
|
||||||
|
t.Fatalf("unexpected candidate observation=%+v", report.Observed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReconcileExplainsDriftWithoutRepairingIt(t *testing.T) {
|
||||||
|
cfg, recorded, observed := baseConfig(t, "singleton_candidate")
|
||||||
|
var handler string
|
||||||
|
cfg.Deployment.Singleton, handler = singletonSettings(t, cfg, recorded)
|
||||||
|
if err := replaceSymlink(cfg.Deployment.Singleton.PreviousLink, observed); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
storeSingletonState(t, cfg, recorded, observed)
|
||||||
|
if err := replaceSymlink(cfg.Deployment.Singleton.CurrentLink, observed); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
writeReleaseIdentity(t, observed, "v0.2.0-preview.3", strings.Repeat("b", 40))
|
||||||
|
candidateHandler, err := renderHandler(cfg.Deployment.Singleton.CaddyHandlerTemplate, cfg.Deployment.Singleton.CandidateAddress)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(handler, candidateHandler, 0o640); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
beforeHandler, err := os.ReadFile(handler)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
operator := &fakeOperator{active: map[string]bool{
|
||||||
|
cfg.Deployment.Singleton.Unit: false,
|
||||||
|
cfg.Service.Name + "-tend-candidate.service": true,
|
||||||
|
}}
|
||||||
|
|
||||||
|
report, err := reconciliationManager(operator).Reconcile(context.Background(), cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if report.Consistent || report.Mutation != "none" {
|
||||||
|
t.Fatalf("report=%+v", report)
|
||||||
|
}
|
||||||
|
codes := map[string]bool{}
|
||||||
|
for _, finding := range report.Findings {
|
||||||
|
codes[finding.Code] = true
|
||||||
|
}
|
||||||
|
for _, code := range []string{"active_release_drift", "active_unit_inactive", "route_handler_drift", "unleased_candidate_active"} {
|
||||||
|
if !codes[code] {
|
||||||
|
t.Fatalf("missing %s in %#v", code, report.Findings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
target, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink)
|
||||||
|
if err != nil || target != observed {
|
||||||
|
t.Fatalf("current=%q err=%v", target, err)
|
||||||
|
}
|
||||||
|
afterHandler, err := os.ReadFile(handler)
|
||||||
|
if err != nil || string(afterHandler) != string(beforeHandler) {
|
||||||
|
t.Fatalf("handler changed err=%v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -18,6 +19,17 @@ type Source struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func Inspect(ctx context.Context, runner process.Runner, dir, branch string) (Source, error) {
|
func Inspect(ctx context.Context, runner process.Runner, dir, branch string) (Source, error) {
|
||||||
|
gitDir, err := gitPath(ctx, runner, dir, "--git-dir")
|
||||||
|
if err != nil {
|
||||||
|
return Source{}, fmt.Errorf("inspect Git directory: %w", err)
|
||||||
|
}
|
||||||
|
commonDir, err := gitPath(ctx, runner, dir, "--git-common-dir")
|
||||||
|
if err != nil {
|
||||||
|
return Source{}, fmt.Errorf("inspect Git common directory: %w", err)
|
||||||
|
}
|
||||||
|
if gitDir != commonDir {
|
||||||
|
return Source{}, errors.New("release packaging does not yet support linked Git worktrees; use a clean standalone clone of the exact pushed commit")
|
||||||
|
}
|
||||||
status, err := runner.Run(ctx, dir, nil, "git", "status", "--porcelain=v1", "--untracked-files=all")
|
status, err := runner.Run(ctx, dir, nil, "git", "status", "--porcelain=v1", "--untracked-files=all")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Source{}, err
|
return Source{}, err
|
||||||
@@ -51,3 +63,18 @@ func Inspect(ctx context.Context, runner process.Runner, dir, branch string) (So
|
|||||||
}
|
}
|
||||||
return Source{Commit: commit, Epoch: epoch}, nil
|
return Source{Commit: commit, Epoch: epoch}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func gitPath(ctx context.Context, runner process.Runner, dir, argument string) (string, error) {
|
||||||
|
out, err := runner.Run(ctx, dir, nil, "git", "rev-parse", argument)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
path := strings.TrimSpace(string(out))
|
||||||
|
if path == "" {
|
||||||
|
return "", errors.New("Git returned an empty path")
|
||||||
|
}
|
||||||
|
if !filepath.IsAbs(path) {
|
||||||
|
path = filepath.Join(dir, path)
|
||||||
|
}
|
||||||
|
return filepath.Clean(path), nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,52 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
package provenance
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type recordingRunner struct {
|
||||||
|
responses map[string][]byte
|
||||||
|
calls []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (runner *recordingRunner) Run(_ context.Context, _ string, _ map[string]string, name string, args ...string) ([]byte, error) {
|
||||||
|
key := name + " " + strings.Join(args, " ")
|
||||||
|
runner.calls = append(runner.calls, key)
|
||||||
|
response, ok := runner.responses[key]
|
||||||
|
if !ok {
|
||||||
|
return nil, errors.New("unexpected command: " + key)
|
||||||
|
}
|
||||||
|
return response, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectAcceptsStandaloneExactPushedCheckout(t *testing.T) {
|
||||||
|
commit := strings.Repeat("a", 40)
|
||||||
|
runner := &recordingRunner{responses: map[string][]byte{
|
||||||
|
"git rev-parse --git-dir": []byte(".git\n"),
|
||||||
|
"git rev-parse --git-common-dir": []byte(".git\n"),
|
||||||
|
"git status --porcelain=v1 --untracked-files=all": nil,
|
||||||
|
"git rev-parse HEAD": []byte(commit + "\n"),
|
||||||
|
"git ls-remote --exit-code origin refs/heads/main": []byte(commit + "\trefs/heads/main\n"),
|
||||||
|
"git show -s --format=%ct " + commit: []byte("1720000000\n"),
|
||||||
|
}}
|
||||||
|
result, err := Inspect(context.Background(), runner, "/source", "main")
|
||||||
|
if err != nil || result.Commit != commit || result.Epoch != 1720000000 {
|
||||||
|
t.Fatalf("result=%+v err=%v", result, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInspectExplainsUnsupportedLinkedWorktreeBeforeRemoteOrBuildWork(t *testing.T) {
|
||||||
|
runner := &recordingRunner{responses: map[string][]byte{
|
||||||
|
"git rev-parse --git-dir": []byte("/repo/.git/worktrees/release\n"),
|
||||||
|
"git rev-parse --git-common-dir": []byte("/repo/.git\n"),
|
||||||
|
}}
|
||||||
|
_, err := Inspect(context.Background(), runner, "/source", "main")
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "linked Git worktrees") || len(runner.calls) != 2 {
|
||||||
|
t.Fatalf("calls=%#v err=%v", runner.calls, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
package state
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/netip"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const CandidateLeaseSchemaVersion = 1
|
||||||
|
|
||||||
|
var (
|
||||||
|
leaseOperationPattern = regexp.MustCompile(`^[0-9a-f]{32}$`)
|
||||||
|
leaseServicePattern = regexp.MustCompile(`^[a-z][a-z0-9-]{1,62}$`)
|
||||||
|
)
|
||||||
|
|
||||||
|
type CandidateLease struct {
|
||||||
|
SchemaVersion int `json:"schema_version"`
|
||||||
|
Service string `json:"service"`
|
||||||
|
OperationID string `json:"operation_id"`
|
||||||
|
Release string `json:"release"`
|
||||||
|
Unit string `json:"unit"`
|
||||||
|
Address string `json:"address"`
|
||||||
|
StartedAt string `json:"started_at"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func CandidateLeasePath(statePath string) string { return statePath + ".candidate-lease.json" }
|
||||||
|
|
||||||
|
func (l CandidateLease) Validate(root, service string) error {
|
||||||
|
if l.SchemaVersion != CandidateLeaseSchemaVersion {
|
||||||
|
return errors.New("candidate lease schema version is unsupported")
|
||||||
|
}
|
||||||
|
if !leaseServicePattern.MatchString(service) || l.Service != service {
|
||||||
|
return errors.New("candidate lease service does not match configuration")
|
||||||
|
}
|
||||||
|
if !leaseOperationPattern.MatchString(l.OperationID) {
|
||||||
|
return errors.New("candidate lease operation ID is invalid")
|
||||||
|
}
|
||||||
|
if err := releaseBelow(root, l.Release); err != nil {
|
||||||
|
return fmt.Errorf("candidate lease release: %w", err)
|
||||||
|
}
|
||||||
|
expectedUnit := service + "-tend-candidate-" + l.OperationID[:12] + ".service"
|
||||||
|
if l.Unit != expectedUnit {
|
||||||
|
return errors.New("candidate lease unit does not match its operation")
|
||||||
|
}
|
||||||
|
address, err := netip.ParseAddrPort(l.Address)
|
||||||
|
if err != nil || !address.Addr().IsLoopback() || address.Port() == 0 {
|
||||||
|
return errors.New("candidate lease address is invalid")
|
||||||
|
}
|
||||||
|
if _, err := time.Parse(time.RFC3339, l.StartedAt); err != nil {
|
||||||
|
return errors.New("candidate lease timestamp is invalid")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func LoadCandidateLease(path, root, service string) (CandidateLease, error) {
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if err != nil {
|
||||||
|
return CandidateLease{}, err
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Size() > 64<<10 {
|
||||||
|
return CandidateLease{}, errors.New("candidate lease must be a bounded regular file")
|
||||||
|
}
|
||||||
|
body, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
return CandidateLease{}, err
|
||||||
|
}
|
||||||
|
decoder := json.NewDecoder(bytes.NewReader(body))
|
||||||
|
decoder.DisallowUnknownFields()
|
||||||
|
var lease CandidateLease
|
||||||
|
if err := decoder.Decode(&lease); err != nil {
|
||||||
|
return CandidateLease{}, fmt.Errorf("decode candidate lease: %w", err)
|
||||||
|
}
|
||||||
|
var extra any
|
||||||
|
if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) {
|
||||||
|
return CandidateLease{}, errors.New("candidate lease contains trailing data")
|
||||||
|
}
|
||||||
|
if err := lease.Validate(root, service); err != nil {
|
||||||
|
return CandidateLease{}, err
|
||||||
|
}
|
||||||
|
return lease, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func StoreCandidateLease(path, root string, lease CandidateLease) error {
|
||||||
|
if err := lease.Validate(root, lease.Service); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info, err := os.Lstat(path); err == nil && (!info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0) {
|
||||||
|
return errors.New("candidate lease path must be a regular file, not a symlink")
|
||||||
|
} else if err != nil && !os.IsNotExist(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
body, err := json.MarshalIndent(lease, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
body = append(body, '\n')
|
||||||
|
dir := filepath.Dir(path)
|
||||||
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
temporary, err := os.CreateTemp(dir, ".tend-candidate-lease-")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
name := temporary.Name()
|
||||||
|
complete := false
|
||||||
|
defer func() {
|
||||||
|
_ = temporary.Close()
|
||||||
|
if !complete {
|
||||||
|
_ = os.Remove(name)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
if err := temporary.Chmod(0o644); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := temporary.Write(body); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := temporary.Sync(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := temporary.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := os.Rename(name, path); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
complete = true
|
||||||
|
return syncDir(dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
func RemoveCandidateLease(path string) error {
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return errors.New("candidate lease path must be a regular file, not a symlink")
|
||||||
|
}
|
||||||
|
if err := os.Remove(path); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return syncDir(filepath.Dir(path))
|
||||||
|
}
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
// SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
package state
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCandidateLeaseRoundTripAndRemoval(t *testing.T) {
|
||||||
|
root := filepath.Join(t.TempDir(), "service")
|
||||||
|
release := filepath.Join(root, "releases", "sha256-"+strings.Repeat("a", 64))
|
||||||
|
if err := os.MkdirAll(release, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
path := CandidateLeasePath(filepath.Join(root, "state.json"))
|
||||||
|
operation := strings.Repeat("d", 32)
|
||||||
|
lease := CandidateLease{SchemaVersion: CandidateLeaseSchemaVersion, Service: "example-site", OperationID: operation, Release: release, Unit: "example-site-tend-candidate-" + operation[:12] + ".service", Address: "127.0.0.1:18092", StartedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)}
|
||||||
|
if err := StoreCandidateLease(path, root, lease); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
loaded, err := LoadCandidateLease(path, root, "example-site")
|
||||||
|
if err != nil || loaded != lease {
|
||||||
|
t.Fatalf("loaded=%+v err=%v", loaded, err)
|
||||||
|
}
|
||||||
|
if err := RemoveCandidateLease(path); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := os.Lstat(path); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("lease remains: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCandidateLeaseRejectsCrossServiceAndSymlink(t *testing.T) {
|
||||||
|
root := filepath.Join(t.TempDir(), "service")
|
||||||
|
release := filepath.Join(root, "releases", "sha256-"+strings.Repeat("a", 64))
|
||||||
|
if err := os.MkdirAll(release, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
operation := strings.Repeat("d", 32)
|
||||||
|
lease := CandidateLease{SchemaVersion: CandidateLeaseSchemaVersion, Service: "example-site", OperationID: operation, Release: release, Unit: "example-site-tend-candidate-" + operation[:12] + ".service", Address: "127.0.0.1:18092", StartedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)}
|
||||||
|
if err := lease.Validate(root, "other-site"); err == nil {
|
||||||
|
t.Fatal("expected service mismatch")
|
||||||
|
}
|
||||||
|
path := CandidateLeasePath(filepath.Join(root, "state.json"))
|
||||||
|
if err := os.Symlink(filepath.Join(root, "elsewhere"), path); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := StoreCandidateLease(path, root, lease); err == nil {
|
||||||
|
t.Fatal("expected symlink refusal")
|
||||||
|
}
|
||||||
|
if err := RemoveCandidateLease(path); err == nil {
|
||||||
|
t.Fatal("expected symlink removal refusal")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCandidateLeasePreservesSchemaOneDeploymentState(t *testing.T) {
|
||||||
|
root := filepath.Join(t.TempDir(), "service")
|
||||||
|
release := filepath.Join(root, "releases", "sha256-"+strings.Repeat("a", 64))
|
||||||
|
if err := os.MkdirAll(release, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
at := time.Unix(1, 0).UTC().Format(time.RFC3339)
|
||||||
|
statePath := filepath.Join(root, "state.json")
|
||||||
|
record := Record{SchemaVersion: SchemaVersion, Strategy: "singleton_candidate", DesiredRelease: release, CandidateRelease: release, ActiveSlot: "singleton", ActiveRelease: release, LastAttemptRelease: release, LastAttemptOutcome: "running", LastAttemptAt: at, UpdatedAt: at}
|
||||||
|
if err := Store(statePath, root, record); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
operation := strings.Repeat("d", 32)
|
||||||
|
lease := CandidateLease{SchemaVersion: CandidateLeaseSchemaVersion, Service: "example-site", OperationID: operation, Release: release, Unit: "example-site-tend-candidate-" + operation[:12] + ".service", Address: "127.0.0.1:18092", StartedAt: at}
|
||||||
|
if err := StoreCandidateLease(CandidateLeasePath(statePath), root, lease); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body, err := os.ReadFile(statePath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !bytes.Contains(body, []byte(`"schema_version": 1`)) || bytes.Contains(body, []byte("candidate_operation_id")) || bytes.Contains(body, []byte("candidate_unit")) {
|
||||||
|
t.Fatalf("deployment state contract changed: %s", body)
|
||||||
|
}
|
||||||
|
loaded, err := Load(statePath, root, "singleton_candidate")
|
||||||
|
if err != nil || loaded.SchemaVersion != 1 || loaded.CandidateRelease != release {
|
||||||
|
t.Fatalf("loaded=%+v err=%v", loaded, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -17,7 +17,7 @@ func TestStoreLoadRoundTripAndRejectSymlink(t *testing.T) {
|
|||||||
}
|
}
|
||||||
path := filepath.Join(root, "state.json")
|
path := filepath.Join(root, "state.json")
|
||||||
at := time.Unix(1, 0).UTC().Format(time.RFC3339)
|
at := time.Unix(1, 0).UTC().Format(time.RFC3339)
|
||||||
record := Record{SchemaVersion: 1, Strategy: "singleton_candidate", DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, LastAttemptRelease: release, LastAttemptOutcome: "succeeded", LastAttemptAt: at, UpdatedAt: at}
|
record := Record{SchemaVersion: SchemaVersion, Strategy: "singleton_candidate", DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, LastAttemptRelease: release, LastAttemptOutcome: "succeeded", LastAttemptAt: at, UpdatedAt: at}
|
||||||
if err := Store(path, root, record); err != nil {
|
if err := Store(path, root, record); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -42,13 +42,13 @@ func TestStoreLoadRoundTripAndRejectSymlink(t *testing.T) {
|
|||||||
func TestRecordRequiresCandidateForRunningAttempt(t *testing.T) {
|
func TestRecordRequiresCandidateForRunningAttempt(t *testing.T) {
|
||||||
root := filepath.Join(t.TempDir(), "service")
|
root := filepath.Join(t.TempDir(), "service")
|
||||||
release := filepath.Join(root, "releases", "sha256-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
|
release := filepath.Join(root, "releases", "sha256-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
|
||||||
record := Record{SchemaVersion: 1, Strategy: "singleton_candidate", DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, LastAttemptRelease: release, LastAttemptOutcome: "running", LastAttemptAt: time.Unix(1, 0).UTC().Format(time.RFC3339), UpdatedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)}
|
record := Record{SchemaVersion: SchemaVersion, Strategy: "singleton_candidate", DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, LastAttemptRelease: release, LastAttemptOutcome: "running", LastAttemptAt: time.Unix(1, 0).UTC().Format(time.RFC3339), UpdatedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)}
|
||||||
if err := record.Validate(root, "singleton_candidate"); err == nil {
|
if err := record.Validate(root, "singleton_candidate"); err == nil {
|
||||||
t.Fatal("expected missing candidate rejection")
|
t.Fatal("expected missing candidate rejection")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
func TestRecordRejectsReleaseOutsideRoot(t *testing.T) {
|
func TestRecordRejectsReleaseOutsideRoot(t *testing.T) {
|
||||||
record := Record{SchemaVersion: 1, Strategy: "singleton_candidate", ActiveSlot: "singleton", ActiveRelease: "/tmp/other/release", UpdatedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)}
|
record := Record{SchemaVersion: SchemaVersion, Strategy: "singleton_candidate", ActiveSlot: "singleton", ActiveRelease: "/tmp/other/release", UpdatedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)}
|
||||||
if err := record.Validate("/opt/example", "singleton_candidate"); err == nil {
|
if err := record.Validate("/opt/example", "singleton_candidate"); err == nil {
|
||||||
t.Fatal("expected path refusal")
|
t.Fatal("expected path refusal")
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -45,6 +45,8 @@ internal/deploy/operator.go
|
|||||||
internal/deploy/operator_test.go
|
internal/deploy/operator_test.go
|
||||||
internal/deploy/ownership_linux.go
|
internal/deploy/ownership_linux.go
|
||||||
internal/deploy/ownership_other.go
|
internal/deploy/ownership_other.go
|
||||||
|
internal/deploy/reconcile.go
|
||||||
|
internal/deploy/reconcile_test.go
|
||||||
internal/deploy/release.go
|
internal/deploy/release.go
|
||||||
internal/deploy/release_mode_linux_test.go
|
internal/deploy/release_mode_linux_test.go
|
||||||
internal/deploy/release_test.go
|
internal/deploy/release_test.go
|
||||||
@@ -54,10 +56,13 @@ internal/packager/packager.go
|
|||||||
internal/packager/packager_test.go
|
internal/packager/packager_test.go
|
||||||
internal/process/run.go
|
internal/process/run.go
|
||||||
internal/provenance/git.go
|
internal/provenance/git.go
|
||||||
|
internal/provenance/git_test.go
|
||||||
internal/serverpolicy/ownership_linux.go
|
internal/serverpolicy/ownership_linux.go
|
||||||
internal/serverpolicy/ownership_other.go
|
internal/serverpolicy/ownership_other.go
|
||||||
internal/serverpolicy/policy.go
|
internal/serverpolicy/policy.go
|
||||||
internal/serverpolicy/policy_test.go
|
internal/serverpolicy/policy_test.go
|
||||||
|
internal/state/lease.go
|
||||||
|
internal/state/lease_test.go
|
||||||
internal/state/state.go
|
internal/state/state.go
|
||||||
internal/state/state_test.go
|
internal/state/state_test.go
|
||||||
internal/transport/protocol.go
|
internal/transport/protocol.go
|
||||||
|
|||||||
Reference in New Issue
Block a user