diff --git a/PUBLIC-SNAPSHOT.json b/PUBLIC-SNAPSHOT.json index 2a35d86..d6d1779 100644 --- a/PUBLIC-SNAPSHOT.json +++ b/PUBLIC-SNAPSHOT.json @@ -1 +1 @@ -{"schema_version":1,"source_commit":"4d7094c8b7c61991bfb67b11fc1558724c874eb2","source_tree":"54a2f74804f7acddf3755d7d4da5b97f5fc28381","source_date_epoch":1786920560,"file_count":68} +{"schema_version":1,"source_commit":"8aab3db43f35e6a49aa497f45d73701b13fc9f32","source_tree":"992132ea4703437dc13ffdbb04a077816c02caf9","source_date_epoch":1787049308,"file_count":76} diff --git a/PUBLIC-SNAPSHOT.sha256 b/PUBLIC-SNAPSHOT.sha256 index ef89ca7..9b1d6e3 100644 --- a/PUBLIC-SNAPSHOT.sha256 +++ b/PUBLIC-SNAPSHOT.sha256 @@ -1 +1 @@ -d30559d85177736bed9cbf4e1fa2e0703e3195d42390539efd098f091d64b983 PUBLIC-SNAPSHOT.json +9c6153ab5b861ec5818591269b2a38a14239f68e89911655aaebec444f54a26d PUBLIC-SNAPSHOT.json diff --git a/README.md b/README.md index 9065cad..feabeee 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ shell hooks. Application-specific data activation remains application-specific. ```text tend check --config tend.json -tend package --config tend.json --version v0.2.0-preview.1 --out dist +tend package --config tend.json --version v0.2.0-preview.2 --out dist tend push --target tend-deploy@host --known-hosts FILE --service NAME --artifact FILE --sha256 HEX --approve-sha256 HEX tend receive --policy /etc/tend/receive-policy.json tend check-server --policy /etc/tend/receive-policy.json @@ -38,6 +38,15 @@ through `/run/lock/tend-deploy.lock`. Builds and transfers remain parallel; only the short Caddy/service activation phase is host-wide. Tend is still a single command, not a daemon. +Each service also keeps a bounded JSONL deployment-event stream and explicit +desired, candidate, active, previous, and last-attempt release identities. The +stream contains only fixed provenance and lifecycle fields; Observatory may +ingest it later, but an event-write failure never blocks deployment or rollback. +After Caddy reload, Tend repeatedly probes the configured canonical HTTPS +origins for the activation window. Blue/green deployments simultaneously keep +checking the previous slot, restoring the prior handler and inactive-slot state +if routed traffic or continuity fails. + Dry-run and digest approval are intentional friction. See the [schema-2 migration guide](docs/SCHEMA_V2_MIGRATION.md) and the [two-service walkthrough](docs/WALKTHROUGH.md). @@ -48,12 +57,22 @@ dependency-free module graph. Tend supports Linux hosts with systemd and Caddy; WSL may be used as a Linux development environment, but native Windows is not a supported execution, deployment, or release-gate platform. -The v0.1 public preview and the restricted v0.2 implementation candidate each -completed maintenance releases and explicit rollback/reactivation for both -Gamertan and the Sandwich Hime website using one reviewed candidate. See the dated +The v0.1 public preview and immutable `v0.2.0-preview.1` each completed +maintenance releases and explicit rollback/reactivation for both Gamertan and +the Sandwich Hime website using one reviewed candidate. See the dated [dogfood evidence](docs/DOGFOOD_EVIDENCE.md) for exact scope and limitations. -The v0.2 preview will not be tagged until one identical binary has deployed and -rolled back both services through the restricted transport. + +The additive `v0.2.0-preview.2` candidate keeps that transport and activation +contract while adding bounded deployment-event JSONL, routed-origin continuity, +rollback annotations, and the operational findings recorded through real +dogfooding. Preview 1 remains unchanged. Preview 2 will not be tagged until one +identical binary has deployed, rolled back, and reactivated Gamertan, the +Sandwich Hime website, and Gamertan Observatory. +Operational friction discovered while applying the same contract to new +services is tracked separately in the +[dogfood friction ledger](docs/DOGFOOD_FRICTION.md). The ledger preserves the +fail-closed behavior and records candidate product improvements instead of +normalizing application-specific deployment workarounds. The canonical public repository begins with a sanitized root snapshot rather than the private development history. diff --git a/RELEASE.md b/RELEASE.md index 25e141c..2b264f9 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -19,13 +19,18 @@ The release tag and attached candidate must be built from the final reviewed source commit. Documentation-only changes after the recorded campaign require one final identical-candidate maintenance pass before tagging. -`v0.2.0-preview.1` is a separate, additive release line. It requires schema 2, +`v0.2.0-preview.1` is an immutable, additive release line. It requires schema 2, the restricted `push`/`receive` transport, root-owned environment-file references, host-wide activation serialization, and HTTPS public-origin smoke. -It may be tagged only after the exact same v0.2 binary successfully deploys, -rolls back, and reactivates both Gamertan and the Sandwich Hime website. EQL is -not part of this generic gate; its SQLite/catalog publication needs a dedicated -adapter rather than arbitrary hooks. +Its exact released source remains unchanged. + +`v0.2.0-preview.2` adds bounded deployment-event JSONL, routed-origin +activation continuity, rollback annotations, and the reviewed operational +friction record. It must preserve every Preview 1 security and release gate. +The exact same Preview 2 binary must deploy, roll back, and reactivate +Gamertan, the Sandwich Hime website, and Gamertan Observatory before the tag is +created. EQL is not part of this generic gate; its SQLite/catalog publication +needs a dedicated adapter rather than arbitrary hooks. `v0.1.0-preview.1` is immutable but withdrawn: its source and module checksums are valid, while a fresh `go install` reports the development identity because diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index d30525d..5f4eaf8 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -50,12 +50,31 @@ probes it, atomically replaces one imported Caddy handler, validates the full Caddy configuration, reloads Caddy, and records the prior active slot. Singleton mode starts the new release in a hardened transient systemd unit on a -separate loopback address, probes it, stops the candidate, changes the current -release pointer, and restarts the installed singleton unit. +separate loopback address and probes it. Tend then validates and atomically +routes the imported Caddy handler to that candidate. While the candidate serves +the canonical origin, Tend changes the current release pointer, restarts and +probes the installed fixed-address unit, validates Caddy again, and routes back +to it. The candidate remains healthy through the activation window and stops +only after the handoff succeeds. After the local post-activation probes, Tend also checks configured HTTPS public -origins. Any failure before state persistence restores the previously observed -Caddy bytes and/or release pointers. Rollback is a separate explicit command +origins throughout a bounded activation window. Blue/green mode also probes the +previous slot, and singleton mode probes the handoff candidate, for health and +readiness throughout that window. Any failure before success is recorded +restores the previously observed Caddy bytes and/or release pointers. State +records desired, candidate, active, previous, and last-attempt +release identities, including failed attempts without claiming they became +active. Rollback is a separate explicit command over the recorded state. It rechecks local health/readiness and public reachability, but deliberately does not apply a future release's content marker to an older release whose routes may differ. Pruning preserves both active and previous releases. + +## Evidence boundary + +Every attempted activation and explicit rollback emits bounded, versioned JSONL +events with an operation ID, service, approved artifact digest, source commit, +release version, phase, slot, elapsed duration, and outcome. Values are +validated rather than copied from command output. The log contains no +environment values, arbitrary process output, HTTP bodies, or secret paths. +Identity, entropy, file, and downstream observability failures are deliberately +best effort and cannot control Tend's deployment or rollback result. diff --git a/docs/DOGFOOD_EVIDENCE.md b/docs/DOGFOOD_EVIDENCE.md index c27139f..76e507b 100644 --- a/docs/DOGFOOD_EVIDENCE.md +++ b/docs/DOGFOOD_EVIDENCE.md @@ -1,8 +1,85 @@ # Preview dogfood evidence This is maintainer-run operational evidence, not an independent audit or a -general reliability claim. It records the acceptance campaign completed on -August 14, 2026 before Tend's first public preview. +general reliability claim. It preserves each dated campaign and its limitations +instead of rewriting earlier observations as though later fixes had already +existed. + +## Final Preview 2 code-candidate campaign — August 18, 2026 + +The final v0.2 Preview 2 implementation candidate completed two explicit +rollback-and-reactivation cycles for Gamertan, the Sandwich Hime website, and +Gamertan Observatory on the production Linux/systemd/Caddy host. The candidate +routed singleton traffic to the already-proven transient process while the +fixed-address installed unit restarted, then restored the canonical upstream +only after loopback and public-origin validation. + +### Assessed Tend candidate + +- Private implementation source commit: + `1fd3b9904c46e817c244196dd5d5a90921ca81a2`. +- Version: `v0.2.0-preview.2`. +- Linux/amd64 binary SHA-256: + `adb4753d4e865775d50d618c999f10dfac9a0de945ccfd9d0b8f2e80d65f4597`. +- Gitea release-candidate archive SHA-256: + `2bdfee2168cb16883d524cf9703adfa6ed5bc2bf44fbbb896993acc5fe6969ec`. +- Toolchain: Go 1.26.6, `CGO_ENABLED=0`, `-trimpath`. +- Trusted verification run 286 and release-candidate run 287 passed. The + archive's checksums, SPDX SBOM, embedded version, commit, clean VCS state, + target, and Go build information were independently rechecked before host + installation. + +### Application maintenance artifacts + +- Gamertan archive SHA-256: + `4700b075640b8b2fb5c17e0e02cf8d96ee67ceee10fe76d108c8b411983a88aa`. +- Sandwich Hime website archive SHA-256: + `46b4da41cf6703fb8818e7d25e3a9c13e57cf700f5608b1888c4adb3352e4d38`. +- Observatory preview 12 archive SHA-256: + `9ef0ddd8ec25d8fb75d6a6887e3ba874df7ebba16d3254f6250fcc646f4fd7f4`. + +Every service ended with the intended current release active and the older +release retained as the explicit rollback target. Gamertan returned to its +green slot; both singleton services returned to their fixed addresses. +Candidate ports and the shared lock were free afterward. Caddy and all five +installed application units were active with zero restarts and no failed +units. + +### Continuity and deployment evidence + +A seven-minute workstation probe sampled the Gamertan origin, both Sandwich +Hime origins, Observatory, and EQL health 1,606 times each throughout the +campaign. It observed no HTTP failure status. One simultaneous client-side +disconnect affected all five destinations during a reload. A controlled +90-second replay therefore observed the same validated no-content-change Caddy +reload from both the workstation and an independent Linux host. The Linux host +recorded 450 successful responses and zero failures for every origin; the +workstation alone repeated one common-mode URL transport error across every +destination. Caddy retained the same PID with zero restarts and continued +serving unrelated requests. The common-mode workstation event is recorded as +an observer-path limitation, not server downtime. + +The authoritative deployment-event files finished at 18 Gamertan events, 14 +Sandwich Hime events, and 16 Observatory events. Observatory's agent cursor for +each stream exactly equalled the corresponding file size, proving complete +consumption. The agent and applications reported no warning-or-higher journal +entries during the campaign. Representative public routes returned HTTP 200, +and the EQL origin remained healthy. + +### Findings closed before this campaign + +Two earlier pre-activation artifacts exposed a mode-restoration defect under a +hardened root umask. They failed before route or pointer mutation. Tend now +reapplies validated archive modes explicitly and tests extraction under umask +`0077`. + +The older singleton strategy briefly exposed an unavailable fixed upstream and +produced transient Observatory-agent `502`s. The final candidate's routed +handoff removed that failure in repeated production activation and rollback. + +This evidence update changes VCS build metadata but not deployment logic. The +release policy therefore still requires one last maintenance pass with the +exact evidence-bearing candidate before the signed public tag is created. ## Restricted multi-service campaign — August 16, 2026 diff --git a/docs/DOGFOOD_FRICTION.md b/docs/DOGFOOD_FRICTION.md new file mode 100644 index 0000000..1c5ed96 --- /dev/null +++ b/docs/DOGFOOD_FRICTION.md @@ -0,0 +1,328 @@ +# Dogfood friction ledger + +This document records friction observed through August 18, 2026 while using +Tend for real maintenance releases. Friction is evidence about the product: it +should become either a clearer contract, safer automation, or an explicit +non-goal. It must not become application-specific shell lore. + +The findings below are maintainer observations, not implemented promises. Tend +continues to fail closed while an option is being designed. In particular, +there is no `--skip-remote` packaging escape hatch, no arbitrary deployment +hook, and no relaxation of the host-wide activation lock. + +## Design standard + +A Tend workflow should make the secure path the short, documented path: + +- source, artifact, approval, configuration, and active-release identities are + explicit and inspectable; +- credentials are narrowly scoped, briefly available, and absent from + artifacts and logs; +- application validation is declarative, bounded, and cannot become a shell; +- first installation, maintenance, activation, rollback, and pruning are + distinct operations; +- every mutation has a recorded prior state and a tested restoration path; +- unrelated services may build and prepare candidates concurrently, while the + shared Caddy activation boundary remains serialized. + +## Observed findings + +| ID | Status | Finding | Current safe behavior | Candidate direction | +| --- | --- | --- | --- | --- | +| F-01 | Workflow mitigation | CI packaging needs proof that the exact commit was pushed. | Packaging verifies the configured remote and fails if Git cannot authenticate. The trusted workflow retains only its job-scoped read-only checkout credential. | Make the proof credential or an authenticated source attestation an explicit Tend input and evidence boundary. | +| F-02 | Open | Application-owned unit and configuration changes are outside the binary activation transaction. | Operators stage, validate, back up, and restore those files separately. | Add an allowlisted configuration transaction or record and validate exact configuration digests. | +| F-03 | Open | A singleton candidate may not reproduce the installed unit's arguments and application configuration. | It receives the production environment file and an isolated listen override; activation still fails closed. | Add a bounded application preflight and explicit, validated candidate invocation. | +| F-04 | Partially resolved | Artifact production, review, approval, transfer, and activation require several identity checks. | v0.2 standardizes the evidence bundle and restricted transfer; the operator still repeats the approved digest intentionally. | Add offline inspection and approval ergonomics without combining build authority and production authority. | +| F-05 | Partially resolved | Tend maintenance assumes an existing adopted service and current release. | `check-server` validates a prepared host, while first installation remains a separately reviewed operator procedure. | Define an explicit `install` or `adopt` transaction rather than silently treating bootstrap as maintenance. | +| F-06 | Partially resolved | Binary rollback can be unsafe when service configuration has changed incompatibly. | v0.2 records desired, candidate, active, previous, and last-attempt releases, but external configuration compatibility is operator-owned. | Bind non-secret configuration identities and preflight results to release state. | +| F-07 | Open | `GOPROXY=off` does not prove that every module metadata lookup is available locally. | Packaging stops before artifact creation when Go cannot resolve the complete pinned module graph. | Separate checksum-verified dependency resolution from a network-disabled, cache-completeness-checked build stage. | +| F-08 | Open | Packaging an otherwise clean pushed commit from a linked Git worktree fails Go's required VCS-status stamp. | Tend stops before artifact creation; package the same exact commit from a clean standalone clone. | Detect linked worktrees during `check`/`package`, explain the supported source shape, and assess a provenance-preserving worktree build that does not weaken `-buildvcs=true`. | +| F-09 | Resolved in Preview 2 | Restarting a fixed-address singleton briefly exposed Caddy to an unavailable upstream. | Tend now routes the imported handler to the candidate, restarts and proves the installed unit behind that handoff, then routes back while the candidate remains healthy through the activation window. | Preserve the production regression campaign and failure-injection matrix. | +| F-10 | Resolved in Preview 2 | A hardened root umask could make an extracted application binary executable only by root. | Tend reapplies every validated archive mode explicitly; extraction is tested under umask `0077`. | Preserve the mode and non-root candidate tests. | +| F-11 | Evidence practice | A single external observer can report common-mode client or network errors as apparent multi-service downtime. | Production campaigns retain server state and use an independent observer before classifying a continuity failure. | Standardize multi-vantage continuity evidence without making an observability dependency part of deployment authority. | + +## F-01: pushed-commit proof in CI + +### Observation + +During an Observatory release-candidate run, verification intentionally checked +out source without persisted credentials. Tend later attempted to prove that +`HEAD` existed on the configured private Gitea branch and Git could not +authenticate. Packaging stopped before producing an artifact. This was the +correct fail-closed outcome, but the credential lifecycle was not obvious from +the workflow contract. + +The trusted workflow uses a job-scoped read-only checkout credential so Tend +can perform the independent remote proof. It is not a long-lived repository or +deployment credential, and it is removed by checkout cleanup at the end of the +job. Production credentials remain unavailable to the build job. + +### Options to assess + +1. Support an explicit read-only Git credential file or credential helper for + package-time proof. Never accept a token in arguments, configuration, + manifests, artifacts, or logs. +2. Accept an authenticated CI source-attestation document binding repository, + branch, commit, tree, workflow identity, and event identity. Define which + CI issuers are trusted and preserve the attestation with release evidence. +3. Accept a signed, pre-verified source bundle whose identity and policy can be + checked without network access. + +Do not add a generic skip flag. An unavailable proof must remain a packaging +failure unless an equally strong proof mode was selected explicitly. + +## F-02: application configuration is not binary activation + +### Observation + +Tend can transact an immutable binary, release pointers, a systemd restart, a +validated Caddy handler, health checks, and rollback state. It does not +currently transact application configuration, systemd unit changes, credential +bindings, or server-local secret files. A first deployment that changes these +files therefore needs a separate backup, validation, installation, and +restoration procedure. + +This boundary is safe but easy to overlook: restoring the previous binary does +not restore an incompatible unit or application configuration. + +### Options to assess + +1. Add strict managed-file entries with exact source and destination paths, + content digest, owner, group, mode, and validation type. Permit only regular + files beneath configured roots; reject symlinks and unknown destinations. +2. Add a separate `tend configure` transaction that backs up, atomically + replaces, validates, and restores supported systemd/Caddy/application files. +3. Keep configuration externally managed, but require Tend to record desired + and active configuration digests and prove candidate/rollback compatibility. + +None of these options should permit arbitrary shell commands. Secret values +remain referenced server-side and must never enter release artifacts or state. + +## F-03: candidate fidelity + +### Observation + +A hardened singleton candidate receives the configured production environment +file and Tend's isolated listen-address override. It does not currently model +the installed systemd unit's complete argument vector. An application whose +configuration path is supplied by unit arguments may therefore start a +candidate with defaults instead of the intended production configuration. +That preserves live-state isolation, but it may prove only executable startup. +A missing credential, incompatible configuration field, filesystem permission, +or data migration requirement can then surface at activation time. + +### Options to assess + +- Define a fixed application preflight command or protocol that validates the + production configuration, credentials, permissions, and data compatibility + without binding the live port or mutating live state. +- Add a strict candidate argument vector to schema 2. Validate each argument, + reject secret values and paths outside the application contract, and pass it + directly to systemd without a shell. +- Permit an allowlisted `check` argument vector, never a shell string, with + bounded time/output and an explicitly non-mutating application contract. +- Record which checks ran against the binary alone and which ran against the + actual production configuration so the evidence cannot overstate coverage. + +## F-04: artifact review and approval ergonomics + +### Observation + +The secure flow deliberately separates build, review, digest approval, +transfer, candidate validation, and activation. v0.2 now produces a consistent +archive, manifest, SBOM, and checksum set, then transfers one exact artifact +through its restricted receiver. In practice, operators still need a +predictable way to discover that CI artifact, inspect it, approve exactly one +digest, and retain the review evidence. Without a first-class review path, +correct manual steps are easy to reconstruct differently for each application. + +### Options to assess + +- Add `tend inspect` for offline, non-mutating verification and a concise human + and JSON summary of source, build, dependency, and archive identities. +- Add an approval record that binds the artifact digest, service, target, + approver, and expiry without containing a credential. +- Standardize one Gitea artifact layout and documented download-to-activation + workflow. Keep production credentials unavailable to verification jobs. + +Approval must remain explicit; better ergonomics must not turn a successful +build into an automatic production mutation. + +## F-05: first installation and adoption + +### Observation + +The maintenance workflow expects an installed service, valid configuration, +and a current release pointer. `check-server` validates the restricted receiver +policy and prepared service boundary, but it does not create them. Observatory +bootstrap therefore required operator-managed service account, directories, +credentials, unit, configuration, and an initial current release before Tend +could own later maintenance safely. + +### Options to assess + +- `tend install`: a deliberately broader, separately approved transaction with + a strict schema and complete rollback of every supported created object. +- `tend adopt`: validate an existing service and release, copy or identify its + immutable artifact, establish state, and refuse ambiguous ownership. +- Keep bootstrap out of Tend, but ship a versioned acceptance checklist and a + machine-readable `check-server` result that maintenance can require. + +Installation and adoption must not be inferred from a missing state file. + +## F-06: release and configuration identity + +### Observation + +Content-addressed releases and v0.2 state make desired, candidate, active, +previous, and last-attempt binary identities clear. A live service is still a +combination of its binary, application configuration, credential bindings, +unit, routing fragment, and sometimes data schema. Tend cannot yet fully +explain that combined identity or determine whether a retained binary is +compatible with the current external configuration. + +### Options to assess + +- Extend the existing desired, candidate, active, previous, and last-attempt + release state with non-secret configuration and unit digests. +- Require rollback compatibility declarations or read-only application + preflight before changing traffic. +- Expose the identities and last validation results through + `tend status --json` for deployment evidence and future Observatory + ingestion. + +Configuration records contain digests and approved metadata only—not secret +contents. + +## F-07: offline module-cache completeness + +### Observation + +An exact Observatory package attempt used `GOPROXY=off` and a previously used +module cache. Source archives for the application dependencies were present, +but Go still needed several module metadata records while Tend enumerated the +complete build graph. Go refused the lookup and Tend stopped before building +or writing an artifact. Re-enabling the checksum-verified public proxy supplied +the missing metadata; the resulting package was byte-identical to the trusted +CI candidate. + +This is safe failure, but `GOPROXY=off` alone is not evidence of a hermetic +build. A cache can be partially populated even when ordinary builds happen to +succeed. + +### Options to assess + +- Add a resolver stage that runs with the pinned toolchain, proxy, and checksum + database, emits the complete module inventory, and materializes a bounded, + read-only cache for the builder. +- Run Tend's package stage with networking disabled and require every module, + checksum, source archive, and metadata record to come from that reviewed + cache. +- Add a non-mutating cache-completeness check that reports missing module + identities before the expensive double build. +- Consider a strictly verified vendored-source mode where repository size and + update review are acceptable; do not silently change dependency modes. + +Do not treat `GONOSUMDB`, `GOPRIVATE`, or a proxy bypass as an offline-build +solution. They alter verification or routing policy rather than proving cache +completeness. + +## F-08: linked-worktree VCS stamping + +### Observation + +An exact clean, pushed Observatory main commit passed verification and public +snapshot isolation from a detached linked Git worktree. Tend accepted its +source and remote provenance, then Go 1.26.6 stopped both `tend package` and an +equivalent direct build at the required `-buildvcs=true` step with `error +obtaining VCS status: exit status 128`. Ordinary Git status and commit queries +from the same worktree succeeded. + +A fresh standalone SSH clone of the identical commit packaged successfully +twice. Both archives were byte-identical and carried the expected commit, Go +version, VCS settings, checksums, manifest, and SBOM. This isolates the failure +to the linked-worktree build shape rather than the application source or module +graph. + +The failure is safe: Tend did not create a partial artifact and must not switch +to `-buildvcs=false`, because the package gate independently verifies the +embedded VCS revision and clean state. Until the interaction is resolved, use +a clean standalone clone for release packaging. + +### Options to assess + +- Detect a `.git` indirection file during `tend check` and fail before the + expensive double build with a precise standalone-clone instruction. +- Reproduce the interaction in a package integration test against the minimum + supported Go release and determine whether it is a Go toolchain limitation + or an invocation/environment defect. +- If linked worktrees can be supported, require the resulting build record to + carry the exact expected `vcs.revision` and `vcs.modified=false`; do not + synthesize those settings or disable VCS stamping. +- Consider an explicit, signed source-bundle input as part of the broader + source-attestation design. It must remain at least as strong as current + pushed-commit proof. + +## F-09: singleton traffic continuity + +### Observation + +During the August 18 Observatory maintenance exercise, the transient candidate +passed health, readiness, and content checks. Tend then stopped that candidate, +changed the singleton pointer, and restarted the installed fixed-address unit. +Caddy still targeted the fixed address during that restart, so the Observatory +agent observed a small number of transient HTTP `502` responses. Its durable +spool retried successfully and no accepted telemetry was lost, but the routed +origin was not continuously available. + +The corrected activation contract treats the candidate as a traffic handoff, +not merely a preflight process. Tend validates and routes the imported Caddy +handler to the candidate before changing the release pointer. It restarts and +probes the fixed-address unit without public traffic, routes back only after +that unit passes, and keeps the candidate healthy throughout the bounded +activation window. Any failure restores the former pointer and exact handler +bytes. If restoration itself cannot be completed, Tend leaves the proven +candidate routed and running for explicit operator recovery instead of causing +a known outage. + +Regression tests cover successful handoff, restart failure, public-origin +failure during the activation window, pointer and handler restoration, Caddy +validation/reload boundaries, and candidate cleanup. Final production evidence +must still repeat deploy, rollback, and reactivation with the exact release +binary before this finding is treated as released. + +## F-10: archive modes under a hardened umask + +### Observation + +The first two August 18 maintenance candidates stopped before activation. The +release archives correctly recorded executable mode `0755`, but extraction by +root under umask `0077` left the installed application binary mode `0700`. +The unprivileged transient candidate could not execute it. Tend emitted failed +candidate evidence, retained the prior release and route, and did not expose +the failed binary to traffic. + +Extraction now reapplies the already validated archive mode after file content +is closed. A Linux regression test sets umask `0077`, extracts the release, and +requires the installed binary to remain executable by the service identity. +The successful maintenance campaign used that corrected Tend binary. + +## Prioritization + +The recommended implementation order is: + +1. final production proof of the singleton traffic handoff; +2. a checksum-verified resolver and network-disabled package contract; +3. an early linked-worktree diagnostic and a provenance-preserving support + decision; +4. `tend inspect` and a standardized CI artifact/approval contract; +5. exact release plus configuration identity in status and state; +6. a bounded application preflight contract; +7. restricted transfer and receive with host policy; +8. explicit adoption for existing services; +9. managed configuration only after its restoration and failure-injection + model is as strong as binary activation. + +Friction entries should be updated with the implementing version, tests, and +dogfood evidence when resolved. Resolved entries remain in this ledger so the +reason for the security boundary is not lost. diff --git a/docs/SCHEMA_V2_MIGRATION.md b/docs/SCHEMA_V2_MIGRATION.md index d765e66..be2cdb0 100644 --- a/docs/SCHEMA_V2_MIGRATION.md +++ b/docs/SCHEMA_V2_MIGRATION.md @@ -14,12 +14,19 @@ been exercised. 5. Set every service's `deployment.lock_file` to `/run/lock/tend-deploy.lock`. 6. Add one or more query-free HTTPS `deployment.public_smoke` checks. -7. Update installed systemd units to read the same environment file as the +7. Add a per-service `deployment.event_log` below its release root and a + bounded `deployment.activation_window_seconds` value. Keep the log + root-owned and grant collectors read access explicitly. +8. For singleton services, add the full Caddy configuration, imported handler, + and one-upstream handler-template paths. The template must be reviewed and + contain exactly one `{{UPSTREAM}}` marker so Tend can keep traffic on the + candidate while the fixed-address unit restarts. +9. Update installed systemd units to read the same environment file as the transient candidate. -8. Install a root-owned `0600` receive policy mapping each service name to its +10. Install a root-owned `0600` receive policy mapping each service name to its exact configuration and artifact-size ceiling. -9. Run `tend check-server` as root before accepting a transfer. -10. Validate, activate, rollback, and reactivate one service at a time. Confirm +11. Run `tend check-server` as root before accepting a transfer. +12. Validate, activate, rollback, and reactivate one service at a time. Confirm unrelated services never restart. Tend does not discover `.env`, infer old values, rewrite a production file, or diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md index aecad43..f9247c2 100644 --- a/docs/THREAT_MODEL.md +++ b/docs/THREAT_MODEL.md @@ -7,11 +7,17 @@ - Release roots, state, pointers, and Caddy files reject symlink substitution at their checked boundaries. - Configuration is strict JSON and is never interpolated into a shell command. -- Candidate health is established before traffic or the singleton current +- Candidate health is established before traffic or any current release pointer changes. New deployments also satisfy configured content smoke checks; rollback uses health and readiness because future-release content markers are not valid requirements for an older retained release. - Caddy configuration validates before reload. +- Canonical routed origins and the previous blue/green slot or singleton + handoff candidate remain under probe for the configured activation window; a + failure restores the old handler and release pointer. +- Desired, candidate, active, previous, and failed-attempt identities remain + distinct in state. Bounded deployment events contain no arbitrary command + output or environment values and cannot block deployment. - An activation failure restores the previously observed state. - Active and previous releases survive pruning. - The restricted receiver accepts one versioned bounded stream, one allowlisted diff --git a/docs/WALKTHROUGH.md b/docs/WALKTHROUGH.md index c199cb0..66ebee3 100644 --- a/docs/WALKTHROUGH.md +++ b/docs/WALKTHROUGH.md @@ -9,10 +9,14 @@ binary at `/usr/local/bin/tend`; it does not need Git or Go. 1. Create `/etc/tend/services`, `/etc/tend/environment`, and `/var/lib/tend/incoming`. The incoming and environment directories are root-owned mode `0700`. -2. Install one schema-2 file per service and one root-owned mode-`0600` +2. Install one schema-2 file per service, one per-service deployment-event log, + and one root-owned mode-`0600` environment file per service. A singleton's shared environment file must not define its configured listen key; its installed unit owns the live - address and Tend overrides only the transient candidate. + address and Tend overrides only the transient candidate. Give each + singleton an imported Caddy handler and a root-owned handler template with + exactly one `{{UPSTREAM}}` marker; this is the bounded traffic handoff while + its fixed-address unit restarts. 3. Install the receive policy, forced `authorized_keys` entry, and exact sudoers rule from `examples/server/` after replacing every placeholder. The sudoers fragment preserves only `SSH_ORIGINAL_COMMAND`; the root receiver requires @@ -46,9 +50,18 @@ services share Caddy. Tend does not stop the other application. ## Failure and recovery exercises -- Change a candidate marker: activation must fail before state is stored. +- Change a candidate marker: activation must fail, preserve the active release, + and record the failed attempt without calling the candidate active. - Make a Caddy template invalid: validation must fail and restore prior bytes. -- Make the public marker unavailable: Tend must restore the former slot/pointer. +- Make the public marker fail after an initially successful request: the + activation window must catch the transient routed failure and restore the + former slot/pointer. +- For a singleton, verify repeated canonical-origin requests remain successful + while Tend routes to the candidate, restarts the fixed-address unit, and + returns traffic to it. Inject failure at both Caddy reloads and require the + prior handler and pointer to be restored. +- Stop the previous blue/green slot during the activation window: Tend must + restore the old Caddy handler instead of accepting reduced continuity. - Run `tend rollback --activate` for one service and verify the other service's units, pointers, and public origin did not change. - Interrupt a transfer: no release becomes active and the incomplete incoming diff --git a/examples/README.md b/examples/README.md index 6ee412b..a1d4815 100644 --- a/examples/README.md +++ b/examples/README.md @@ -18,13 +18,24 @@ the isolated candidate address. Secret values never enter `tend.json`. The singleton example follows the same split: its shared root-only environment file omits the configured listen key, the installed unit owns the live address, and Tend supplies only the transient candidate address. This prevents a shared -environment file from overriding the isolated candidate port. +environment file from overriding the isolated candidate port. Its imported +Caddy handler is also managed from one reviewed template. Tend temporarily +routes the canonical origin to the proven candidate while the fixed-address +unit restarts, then returns traffic to that unit only after it passes its local +checks. Production configuration belongs outside the source checkout, owned by root, and not group- or world-writable. The Caddy handler template is an entire imported handler fragment; the enclosing site, matchers, and routing precedence remain operator-owned. +`event_log` is a per-service, root-owned JSONL evidence stream below that +service's release root. Grant an Observatory agent read access explicitly; do +not make the release root broadly readable. `activation_window_seconds` keeps +canonical routed probes active after Caddy reload and keeps the previous +blue/green slot—or the singleton handoff candidate—under health/readiness +observation until the activation is recorded. + `server/` demonstrates the schema-2 receive policy, forced OpenSSH command, restricted sudo entry, two independent service configurations, and secret-file placement. The values are placeholders, not an installation script. diff --git a/examples/blue-green/tend.json b/examples/blue-green/tend.json index b1b9a06..641a1ff 100644 --- a/examples/blue-green/tend.json +++ b/examples/blue-green/tend.json @@ -7,9 +7,11 @@ "root": "/opt/example-site", "lock_file": "/run/lock/tend-deploy.lock", "state_file": "/opt/example-site/tend-state.json", + "event_log": "/opt/example-site/deployment-events.jsonl", "health_path": "/healthz", "readiness_path": "/readyz", "candidate_timeout_seconds": 30, + "activation_window_seconds": 10, "smoke": [{ "path": "/", "contains": "Example site" }], "public_smoke": [{ "url": "https://example.test/", "contains": "Example site" }], "blue_green": { diff --git a/examples/server/caddy/docs-site.template b/examples/server/caddy/docs-site.template new file mode 100644 index 0000000..aa4fe52 --- /dev/null +++ b/examples/server/caddy/docs-site.template @@ -0,0 +1,2 @@ +# Managed by Tend. The enclosing site and route matchers remain operator-owned. +reverse_proxy {{UPSTREAM}} diff --git a/examples/server/caddy/example-site.template b/examples/server/caddy/example-site.template new file mode 100644 index 0000000..aa4fe52 --- /dev/null +++ b/examples/server/caddy/example-site.template @@ -0,0 +1,2 @@ +# Managed by Tend. The enclosing site and route matchers remain operator-owned. +reverse_proxy {{UPSTREAM}} diff --git a/examples/server/services/docs-site.json b/examples/server/services/docs-site.json index c396f31..5e7dc68 100644 --- a/examples/server/services/docs-site.json +++ b/examples/server/services/docs-site.json @@ -7,9 +7,11 @@ "root": "/opt/docs-site", "lock_file": "/run/lock/tend-deploy.lock", "state_file": "/opt/docs-site/tend-state.json", + "event_log": "/opt/docs-site/deployment-events.jsonl", "health_path": "/healthz", "readiness_path": "/readyz", "candidate_timeout_seconds": 30, + "activation_window_seconds": 10, "smoke": [{ "path": "/", "contains": "Documentation" }], "public_smoke": [{ "url": "https://docs.example.test/", "contains": "Documentation" }], "singleton": { @@ -18,7 +20,10 @@ "candidate_address": "127.0.0.1:18102", "listen_env": "DOCS_LISTEN", "current_link": "/opt/docs-site/current", - "previous_link": "/opt/docs-site/previous" + "previous_link": "/opt/docs-site/previous", + "caddy_config": "/etc/caddy/Caddyfile", + "caddy_handler": "/etc/caddy/docs-site-handler.caddy", + "caddy_handler_template": "/etc/tend/caddy/docs-site.template" } } } diff --git a/examples/server/services/example-site.json b/examples/server/services/example-site.json index 2277047..b2f4fcf 100644 --- a/examples/server/services/example-site.json +++ b/examples/server/services/example-site.json @@ -7,9 +7,11 @@ "root": "/opt/example-site", "lock_file": "/run/lock/tend-deploy.lock", "state_file": "/opt/example-site/tend-state.json", + "event_log": "/opt/example-site/deployment-events.jsonl", "health_path": "/healthz", "readiness_path": "/readyz", "candidate_timeout_seconds": 30, + "activation_window_seconds": 10, "smoke": [{ "path": "/", "contains": "Example site" }], "public_smoke": [{ "url": "https://example.test/", "contains": "Example site" }], "singleton": { @@ -18,7 +20,10 @@ "candidate_address": "127.0.0.1:18092", "listen_env": "EXAMPLE_LISTEN", "current_link": "/opt/example-site/current", - "previous_link": "/opt/example-site/previous" + "previous_link": "/opt/example-site/previous", + "caddy_config": "/etc/caddy/Caddyfile", + "caddy_handler": "/etc/caddy/example-site-handler.caddy", + "caddy_handler_template": "/etc/tend/caddy/example-site.template" } } } diff --git a/examples/singleton/caddy-handler.template b/examples/singleton/caddy-handler.template new file mode 100644 index 0000000..aa4fe52 --- /dev/null +++ b/examples/singleton/caddy-handler.template @@ -0,0 +1,2 @@ +# Managed by Tend. The enclosing site and route matchers remain operator-owned. +reverse_proxy {{UPSTREAM}} diff --git a/examples/singleton/tend.json b/examples/singleton/tend.json index 2277047..b2f4fcf 100644 --- a/examples/singleton/tend.json +++ b/examples/singleton/tend.json @@ -7,9 +7,11 @@ "root": "/opt/example-site", "lock_file": "/run/lock/tend-deploy.lock", "state_file": "/opt/example-site/tend-state.json", + "event_log": "/opt/example-site/deployment-events.jsonl", "health_path": "/healthz", "readiness_path": "/readyz", "candidate_timeout_seconds": 30, + "activation_window_seconds": 10, "smoke": [{ "path": "/", "contains": "Example site" }], "public_smoke": [{ "url": "https://example.test/", "contains": "Example site" }], "singleton": { @@ -18,7 +20,10 @@ "candidate_address": "127.0.0.1:18092", "listen_env": "EXAMPLE_LISTEN", "current_link": "/opt/example-site/current", - "previous_link": "/opt/example-site/previous" + "previous_link": "/opt/example-site/previous", + "caddy_config": "/etc/caddy/Caddyfile", + "caddy_handler": "/etc/caddy/example-site-handler.caddy", + "caddy_handler_template": "/etc/tend/caddy/example-site.template" } } } diff --git a/internal/config/config.go b/internal/config/config.go index cd4834d..bc0571b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -56,9 +56,11 @@ type Deployment struct { Root string `json:"root"` LockFile string `json:"lock_file"` StateFile string `json:"state_file"` + EventLog string `json:"event_log"` HealthPath string `json:"health_path"` ReadinessPath string `json:"readiness_path"` CandidateTimeoutSecs int `json:"candidate_timeout_seconds"` + ActivationWindowSecs int `json:"activation_window_seconds"` Smoke []Smoke `json:"smoke"` PublicSmoke []PublicSmoke `json:"public_smoke"` BlueGreen *BlueGreen `json:"blue_green,omitempty"` @@ -91,12 +93,15 @@ type Slot struct { } type Singleton struct { - Unit string `json:"unit"` - Address string `json:"address"` - CandidateAddress string `json:"candidate_address"` - ListenEnv string `json:"listen_env"` - CurrentLink string `json:"current_link"` - PreviousLink string `json:"previous_link"` + Unit string `json:"unit"` + Address string `json:"address"` + CandidateAddress string `json:"candidate_address"` + ListenEnv string `json:"listen_env"` + CurrentLink string `json:"current_link"` + PreviousLink string `json:"previous_link"` + CaddyConfig string `json:"caddy_config"` + CaddyHandler string `json:"caddy_handler"` + CaddyHandlerTemplate string `json:"caddy_handler_template"` } func Load(path string) (Config, error) { @@ -187,12 +192,21 @@ func (c Config) Validate() error { if filepath.Clean(d.StateFile) == filepath.Clean(d.Root) || !within(d.Root, d.StateFile) { return errors.New("deployment.state_file must be below deployment.root") } + if err := safeAbsolute("deployment.event_log", d.EventLog); err != nil { + return err + } + if filepath.Clean(d.EventLog) == filepath.Clean(d.Root) || !within(d.Root, d.EventLog) || filepath.Clean(d.EventLog) == filepath.Clean(d.StateFile) { + return errors.New("deployment.event_log must be a distinct file below deployment.root") + } if !safeHTTPPath(d.HealthPath) || !safeHTTPPath(d.ReadinessPath) { return errors.New("health and readiness paths must be absolute HTTP paths") } if d.CandidateTimeoutSecs < 2 || d.CandidateTimeoutSecs > 300 { return errors.New("candidate_timeout_seconds must be between 2 and 300") } + if d.ActivationWindowSecs < 1 || d.ActivationWindowSecs > 120 { + return errors.New("activation_window_seconds must be between 1 and 120") + } if len(d.Smoke) == 0 || len(d.Smoke) > 32 { return errors.New("deployment.smoke must contain 1 to 32 checks") } @@ -298,6 +312,14 @@ func validateSingleton(root string, s Singleton) error { if s.CurrentLink == s.PreviousLink { return errors.New("current and previous links must differ") } + for label, path := range map[string]string{"caddy_config": s.CaddyConfig, "caddy_handler": s.CaddyHandler, "caddy_handler_template": s.CaddyHandlerTemplate} { + if err := safeAbsolute("deployment.singleton."+label, path); err != nil { + return err + } + } + if filepath.Clean(s.CaddyHandler) == filepath.Clean(s.CaddyHandlerTemplate) { + return errors.New("singleton Caddy handler and template must be different files") + } return nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 12a6164..4f1b965 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -14,8 +14,8 @@ func validConfig() Config { Build: Build{Package: "./cmd/site", Binary: "example-site", Branch: "main"}, Deployment: Deployment{ Strategy: "blue_green", Root: "/opt/example-site", LockFile: SharedLockFile, - StateFile: "/opt/example-site/state.json", HealthPath: "/healthz", ReadinessPath: "/readyz", - CandidateTimeoutSecs: 30, Smoke: []Smoke{{Path: "/", Contains: "Example"}}, + StateFile: "/opt/example-site/state.json", EventLog: "/opt/example-site/deployment-events.jsonl", HealthPath: "/healthz", ReadinessPath: "/readyz", + CandidateTimeoutSecs: 30, ActivationWindowSecs: 10, Smoke: []Smoke{{Path: "/", Contains: "Example"}}, PublicSmoke: []PublicSmoke{{URL: "https://example.test/", Contains: "Example"}}, BlueGreen: &BlueGreen{ CaddyConfig: "/etc/caddy/Caddyfile", CaddyHandler: "/etc/caddy/example.caddy", @@ -34,6 +34,24 @@ func TestValidateAcceptsBlueGreen(t *testing.T) { } } +func TestValidateSingletonRequiresDistinctCaddyHandoffFiles(t *testing.T) { + cfg := validConfig() + cfg.Deployment.Strategy = "singleton_candidate" + cfg.Deployment.BlueGreen = nil + cfg.Deployment.Singleton = &Singleton{ + Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN", + CurrentLink: "/opt/example-site/current", PreviousLink: "/opt/example-site/previous", CaddyConfig: "/etc/caddy/Caddyfile", + CaddyHandler: "/etc/caddy/example-site.caddy", CaddyHandlerTemplate: "/etc/tend/caddy/example-site.template", + } + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + cfg.Deployment.Singleton.CaddyHandlerTemplate = cfg.Deployment.Singleton.CaddyHandler + if err := cfg.Validate(); err == nil { + t.Fatal("expected shared handler/template path to be rejected") + } +} + func TestValidateRejectsHostileValues(t *testing.T) { tests := map[string]func(*Config){ "unknown strategy": func(c *Config) { c.Deployment.Strategy = "shell" }, diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go index 44d3f99..69ec15b 100644 --- a/internal/deploy/deploy.go +++ b/internal/deploy/deploy.go @@ -14,6 +14,7 @@ import ( "time" "gamertan.com/tend/internal/config" + "gamertan.com/tend/internal/eventlog" "gamertan.com/tend/internal/state" ) @@ -29,6 +30,7 @@ type Report struct { Release string `json:"release,omitempty"` ActiveRelease string `json:"active_release,omitempty"` PreviousRelease string `json:"previous_release,omitempty"` + EventWarnings int `json:"event_warnings,omitempty"` } type Status struct { State *state.Record `json:"state,omitempty"` @@ -37,14 +39,18 @@ type Status struct { } type Manager struct { - Operator Operator - Now func() time.Time - Prepare func(config.Config, string, string, string) (string, error) - Inspect func(config.Config, string, string, string) error + Operator Operator + Now func() time.Time + Prepare func(config.Config, string, string, string) (string, error) + Inspect func(config.Config, string, string, string) error + ReadIdentity func(string) (releaseIdentity, error) + OperationID func() (string, error) + AppendEvent func(string, eventlog.Event) error + Sleep func(context.Context, time.Duration) error } func NewManager(operator Operator) Manager { - return Manager{Operator: operator, Now: time.Now, Prepare: prepareRelease, Inspect: inspectArtifact} + return Manager{Operator: operator, Now: time.Now, Prepare: prepareRelease, Inspect: inspectArtifact, ReadIdentity: readReleaseIdentity, OperationID: eventlog.OperationID, AppendEvent: eventlog.Append, Sleep: sleepContext} } func (m Manager) Deploy(ctx context.Context, cfg config.Config, request Request) (Report, error) { @@ -66,10 +72,44 @@ func (m Manager) Deploy(ctx context.Context, cfg config.Config, request Request) if err != nil { return Report{}, err } + started := m.Now() + eventWarnings := 0 + identity, identityErr := m.ReadIdentity(release) + if identityErr != nil { + eventWarnings++ + } + operationID := "" + if m.OperationID != nil { + operationID, err = m.OperationID() + if err != nil { + eventWarnings++ + operationID = "" + } + } + emit := func(phase, slot, outcome string) { + if m.AppendEvent == nil || identityErr != nil || operationID == "" { + return + } + event := eventlog.Event{Version: eventlog.Version, OperationID: operationID, Service: cfg.Service.Name, ArtifactDigest: request.ApprovedSHA256, Commit: identity.Commit, ReleaseVersion: identity.Version, Phase: phase, Slot: slot, DurationMillis: max(0, m.Now().Sub(started).Milliseconds()), Outcome: outcome, ObservedAt: m.Now().UTC().Format(time.RFC3339Nano)} + if eventErr := m.AppendEvent(cfg.Deployment.EventLog, event); eventErr != nil { + eventWarnings++ + } + } record, err := loadOrBootstrap(cfg, m.Now()) if err != nil { return Report{}, err } + attemptAt := m.Now().UTC().Format(time.RFC3339) + record.DesiredRelease = release + record.CandidateRelease = release + record.LastAttemptRelease = release + record.LastAttemptOutcome = "running" + record.LastAttemptAt = attemptAt + record.UpdatedAt = attemptAt + if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil { + return Report{}, err + } + emit("candidate", inactiveSlot(cfg, record), "running") switch cfg.Deployment.Strategy { case "blue_green": err = m.deployBlueGreen(ctx, cfg, record, release) @@ -79,13 +119,64 @@ func (m Manager) Deploy(ctx context.Context, cfg config.Config, request Request) err = errors.New("unsupported strategy") } if err != nil { - return Report{}, err + failed := record + failed.CandidateRelease = "" + failed.LastAttemptOutcome = "failed" + failed.UpdatedAt = m.Now().UTC().Format(time.RFC3339) + _ = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, failed) + emit("activation", inactiveSlot(cfg, record), "failed") + return Report{EventWarnings: eventWarnings}, err } + emit("activation", inactiveSlot(cfg, record), "succeeded") updated, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) if err != nil { return Report{}, err } - return Report{Validated: true, Mutation: "activated", Release: release, ActiveRelease: updated.ActiveRelease, PreviousRelease: updated.PreviousRelease}, nil + return Report{Validated: true, Mutation: "activated", Release: release, ActiveRelease: updated.ActiveRelease, PreviousRelease: updated.PreviousRelease, EventWarnings: eventWarnings}, nil +} + +type releaseIdentity struct { + Version string `json:"version"` + Commit string `json:"commit"` +} + +func readReleaseIdentity(release string) (releaseIdentity, error) { + b, err := os.ReadFile(filepath.Join(release, "RELEASE.json")) + if err != nil { + return releaseIdentity{}, fmt.Errorf("read installed release identity: %w", err) + } + if len(b) > 1<<20 { + return releaseIdentity{}, errors.New("installed release identity is too large") + } + var identity releaseIdentity + if err := json.Unmarshal(b, &identity); err != nil { + return releaseIdentity{}, errors.New("decode installed release identity") + } + if identity.Version == "" || identity.Commit == "" { + return releaseIdentity{}, errors.New("installed release identity is incomplete") + } + return identity, nil +} + +func inactiveSlot(cfg config.Config, record state.Record) string { + if cfg.Deployment.Strategy == "singleton_candidate" { + return "singleton" + } + if record.ActiveSlot == "blue" { + return "green" + } + return "blue" +} + +func sleepContext(ctx context.Context, duration time.Duration) error { + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } } func loadOrBootstrap(cfg config.Config, now time.Time) (state.Record, error) { @@ -103,13 +194,13 @@ func loadOrBootstrap(cfg config.Config, now time.Time) (state.Record, error) { if err != nil { return state.Record{}, fmt.Errorf("bootstrap active slot: %w", err) } - return state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, ActiveSlot: slot, ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil + return state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: slot, ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil case "singleton_candidate": release, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink) if err != nil { return state.Record{}, fmt.Errorf("bootstrap singleton: %w", err) } - return state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, ActiveSlot: "singleton", ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil + return state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil } return state.Record{}, errors.New("unsupported strategy") } @@ -180,7 +271,11 @@ func (m Manager) deployBlueGreen(ctx context.Context, cfg config.Config, record if err = m.probePublic(ctx, cfg, true); err != nil { return fmt.Errorf("public-origin smoke failed: %w", err) } - next := state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, ActiveSlot: inactive, ActiveRelease: release, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, UpdatedAt: m.Now().UTC().Format(time.RFC3339)} + previous := slotConfig(bg, record.ActiveSlot) + if err = m.continuityWindow(ctx, cfg, previous.Address, true); err != nil { + return fmt.Errorf("activation continuity failed: %w", err) + } + next := state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: inactive, ActiveRelease: release, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, LastAttemptRelease: release, LastAttemptOutcome: "succeeded", LastAttemptAt: record.LastAttemptAt, UpdatedAt: m.Now().UTC().Format(time.RFC3339)} if err = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next); err != nil { return err } @@ -188,62 +283,10 @@ func (m Manager) deployBlueGreen(ctx context.Context, cfg config.Config, record } func (m Manager) deploySingleton(ctx context.Context, cfg config.Config, record state.Record, release string) (err error) { - single := *cfg.Deployment.Singleton - candidateUnit := cfg.Service.Name + "-tend-candidate.service" - env := map[string]string{single.ListenEnv: single.CandidateAddress} - binary := filepath.Join(release, cfg.Build.Binary) - if err = m.Operator.StartCandidate(ctx, candidateUnit, binary, cfg.Service.EnvironmentFile, env); err != nil { + if err = m.activateSingletonRelease(ctx, cfg, release, true); err != nil { return err } - defer func() { - stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _ = m.Operator.Stop(stopCtx, candidateUnit) - }() - if err = m.probeAll(ctx, cfg, single.CandidateAddress); err != nil { - return fmt.Errorf("candidate failed: %w", err) - } - oldCurrent, err := resolveReleaseLink(cfg.Deployment.Root, single.CurrentLink) - if err != nil { - return err - } - oldPrevious, previousErr := resolveReleaseLink(cfg.Deployment.Root, single.PreviousLink) - currentChanged := false - previousChanged := false - defer func() { - if err == nil { - return - } - if currentChanged { - _ = replaceSymlink(single.CurrentLink, oldCurrent) - _ = m.Operator.Restart(ctx, single.Unit) - } - if previousChanged { - if previousErr == nil { - _ = replaceSymlink(single.PreviousLink, oldPrevious) - } else { - _ = removeSymlink(single.PreviousLink) - } - } - }() - if err = replaceSymlink(single.PreviousLink, oldCurrent); err != nil { - return err - } - previousChanged = true - if err = replaceSymlink(single.CurrentLink, release); err != nil { - return err - } - currentChanged = true - if err = m.Operator.Restart(ctx, single.Unit); err != nil { - return err - } - if err = m.probeAll(ctx, cfg, single.Address); err != nil { - return fmt.Errorf("post-activation smoke failed: %w", err) - } - if err = m.probePublic(ctx, cfg, true); err != nil { - return fmt.Errorf("public-origin smoke failed: %w", err) - } - next := state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, ActiveSlot: "singleton", ActiveRelease: release, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, UpdatedAt: m.Now().UTC().Format(time.RFC3339)} + next := state.Record{SchemaVersion: 1, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, LastAttemptRelease: release, LastAttemptOutcome: "succeeded", LastAttemptAt: record.LastAttemptAt, UpdatedAt: m.Now().UTC().Format(time.RFC3339)} if err = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next); err != nil { return err } @@ -263,6 +306,21 @@ func (m Manager) Rollback(ctx context.Context, cfg config.Config) (state.Record, if record.PreviousRelease == "" { return state.Record{}, errors.New("no previous release is recorded") } + started := m.Now() + identity, identityErr := m.ReadIdentity(record.PreviousRelease) + digest, digestErr := releaseDigest(record.PreviousRelease) + operationID := "" + if m.OperationID != nil { + operationID, _ = m.OperationID() + } + emit := func(outcome string) { + if m.AppendEvent == nil || identityErr != nil || digestErr != nil || operationID == "" { + return + } + event := eventlog.Event{Version: eventlog.Version, OperationID: operationID, Service: cfg.Service.Name, ArtifactDigest: digest, Commit: identity.Commit, ReleaseVersion: identity.Version, Phase: "rollback", Slot: record.PreviousSlot, DurationMillis: max(0, m.Now().Sub(started).Milliseconds()), Outcome: outcome, ObservedAt: m.Now().UTC().Format(time.RFC3339Nano)} + _ = m.AppendEvent(cfg.Deployment.EventLog, event) + } + emit("running") switch cfg.Deployment.Strategy { case "blue_green": err = m.rollbackBlueGreen(ctx, cfg, record) @@ -272,10 +330,29 @@ func (m Manager) Rollback(ctx context.Context, cfg config.Config) (state.Record, err = errors.New("unsupported strategy") } if err != nil { + emit("failed") return state.Record{}, err } + emit("succeeded") return state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) } + +func releaseDigest(release string) (string, error) { + name := filepath.Base(release) + if !strings.HasPrefix(name, "sha256-") { + return "", errors.New("release is not content addressed") + } + digest := strings.TrimPrefix(name, "sha256-") + if len(digest) != 64 { + return "", errors.New("release digest is invalid") + } + for _, character := range digest { + if !strings.ContainsRune("0123456789abcdef", character) { + return "", errors.New("release digest is invalid") + } + } + return digest, nil +} func (m Manager) rollbackBlueGreen(ctx context.Context, cfg config.Config, record state.Record) (err error) { bg := *cfg.Deployment.BlueGreen slot := slotConfig(bg, record.PreviousSlot) @@ -321,36 +398,148 @@ func (m Manager) rollbackBlueGreen(ctx context.Context, cfg config.Config, recor if err = m.probePublic(ctx, cfg, false); err != nil { return err } - next := state.Record{SchemaVersion: 1, Strategy: record.Strategy, ActiveSlot: record.PreviousSlot, ActiveRelease: record.PreviousRelease, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, UpdatedAt: m.Now().UTC().Format(time.RFC3339)} + next := state.Record{SchemaVersion: 1, Strategy: record.Strategy, DesiredRelease: record.PreviousRelease, ActiveSlot: record.PreviousSlot, ActiveRelease: record.PreviousRelease, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, LastAttemptRelease: record.PreviousRelease, LastAttemptOutcome: "rolled_back", LastAttemptAt: m.Now().UTC().Format(time.RFC3339), UpdatedAt: m.Now().UTC().Format(time.RFC3339)} return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next) } func (m Manager) rollbackSingleton(ctx context.Context, cfg config.Config, record state.Record) (err error) { + if err = m.activateSingletonRelease(ctx, cfg, record.PreviousRelease, false); err != nil { + return err + } + next := state.Record{SchemaVersion: 1, Strategy: record.Strategy, DesiredRelease: record.PreviousRelease, ActiveSlot: "singleton", ActiveRelease: record.PreviousRelease, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, LastAttemptRelease: record.PreviousRelease, LastAttemptOutcome: "rolled_back", LastAttemptAt: m.Now().UTC().Format(time.RFC3339), UpdatedAt: m.Now().UTC().Format(time.RFC3339)} + return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next) +} + +// activateSingletonRelease keeps public traffic on a proven process while the +// installed fixed-address unit changes release. The transient candidate first +// receives traffic, remains healthy through the handoff, and is stopped only +// after Caddy points back to the verified installed unit. +func (m Manager) activateSingletonRelease(ctx context.Context, cfg config.Config, release string, checkMarkers bool) (err error) { single := *cfg.Deployment.Singleton + candidateUnit := cfg.Service.Name + "-tend-candidate.service" + env := map[string]string{single.ListenEnv: single.CandidateAddress} + binary := filepath.Join(release, cfg.Build.Binary) + if err = m.Operator.StartCandidate(ctx, candidateUnit, binary, cfg.Service.EnvironmentFile, env); err != nil { + return err + } + stopCandidate := true + defer func() { + if !stopCandidate { + return + } + stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = m.Operator.Stop(stopCtx, candidateUnit) + }() + probeLocal := m.probeHealthReadiness + if checkMarkers { + probeLocal = m.probeAll + } + if err = probeLocal(ctx, cfg, single.CandidateAddress); err != nil { + return fmt.Errorf("candidate failed: %w", err) + } + + oldHandler, err := os.ReadFile(single.CaddyHandler) + if err != nil { + return fmt.Errorf("read current Caddy handler: %w", err) + } oldCurrent, err := resolveReleaseLink(cfg.Deployment.Root, single.CurrentLink) if err != nil { return err } - if err = replaceSymlink(single.CurrentLink, record.PreviousRelease); err != nil { - return err - } + oldPrevious, previousErr := resolveReleaseLink(cfg.Deployment.Root, single.PreviousLink) + handlerChanged := false + currentChanged := false + previousChanged := false defer func() { - if err != nil { - _ = replaceSymlink(single.CurrentLink, oldCurrent) - _ = m.Operator.Restart(ctx, single.Unit) + if err == nil { + return + } + recoveryErr := error(nil) + if currentChanged { + if restoreErr := replaceSymlink(single.CurrentLink, oldCurrent); restoreErr != nil { + recoveryErr = errors.Join(recoveryErr, restoreErr) + } else if restoreErr = m.Operator.Restart(ctx, single.Unit); restoreErr != nil { + recoveryErr = errors.Join(recoveryErr, restoreErr) + } else if restoreErr = m.probeHealthReadiness(ctx, cfg, single.Address); restoreErr != nil { + recoveryErr = errors.Join(recoveryErr, restoreErr) + } + } + if previousChanged { + var restoreErr error + if previousErr == nil { + restoreErr = replaceSymlink(single.PreviousLink, oldPrevious) + } else { + restoreErr = removeSymlink(single.PreviousLink) + } + recoveryErr = errors.Join(recoveryErr, restoreErr) + } + if handlerChanged && recoveryErr == nil { + if restoreErr := atomicWrite(single.CaddyHandler, oldHandler, 0o644); restoreErr != nil { + recoveryErr = errors.Join(recoveryErr, restoreErr) + } else if restoreErr = m.Operator.ValidateCaddy(ctx, single.CaddyConfig); restoreErr != nil { + recoveryErr = errors.Join(recoveryErr, restoreErr) + } else if restoreErr = m.Operator.ReloadCaddy(ctx); restoreErr != nil { + recoveryErr = errors.Join(recoveryErr, restoreErr) + } + } + if recoveryErr != nil && handlerChanged { + stopCandidate = false + err = errors.Join(err, fmt.Errorf("singleton recovery incomplete; candidate remains routed for operator recovery: %w", recoveryErr)) } }() + + candidateHandler, err := renderHandler(single.CaddyHandlerTemplate, single.CandidateAddress) + if err != nil { + return err + } + if err = atomicWrite(single.CaddyHandler, candidateHandler, 0o644); err != nil { + return err + } + handlerChanged = true + if err = m.Operator.ValidateCaddy(ctx, single.CaddyConfig); err != nil { + return fmt.Errorf("candidate Caddy validation failed: %w", err) + } + if err = m.Operator.ReloadCaddy(ctx); err != nil { + return fmt.Errorf("candidate Caddy reload failed: %w", err) + } + if err = m.probePublic(ctx, cfg, checkMarkers); err != nil { + return fmt.Errorf("candidate public-origin smoke failed: %w", err) + } + + if err = replaceSymlink(single.PreviousLink, oldCurrent); err != nil { + return err + } + previousChanged = true + if err = replaceSymlink(single.CurrentLink, release); err != nil { + return err + } + currentChanged = true if err = m.Operator.Restart(ctx, single.Unit); err != nil { return err } - if err = m.probeHealthReadiness(ctx, cfg, single.Address); err != nil { + if err = probeLocal(ctx, cfg, single.Address); err != nil { + return fmt.Errorf("post-activation smoke failed: %w", err) + } + installedHandler, err := renderHandler(single.CaddyHandlerTemplate, single.Address) + if err != nil { return err } - if err = m.probePublic(ctx, cfg, false); err != nil { + if err = atomicWrite(single.CaddyHandler, installedHandler, 0o644); err != nil { return err } - _ = replaceSymlink(single.PreviousLink, record.ActiveRelease) - next := state.Record{SchemaVersion: 1, Strategy: record.Strategy, ActiveSlot: "singleton", ActiveRelease: record.PreviousRelease, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, UpdatedAt: m.Now().UTC().Format(time.RFC3339)} - return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next) + if err = m.Operator.ValidateCaddy(ctx, single.CaddyConfig); err != nil { + return fmt.Errorf("installed Caddy validation failed: %w", err) + } + if err = m.Operator.ReloadCaddy(ctx); err != nil { + return fmt.Errorf("installed Caddy reload failed: %w", err) + } + if err = m.probePublic(ctx, cfg, checkMarkers); err != nil { + return fmt.Errorf("public-origin smoke failed: %w", err) + } + if err = m.continuityWindow(ctx, cfg, single.CandidateAddress, checkMarkers); err != nil { + return fmt.Errorf("activation continuity failed: %w", err) + } + return nil } func (m Manager) Status(ctx context.Context, cfg config.Config) (Status, error) { @@ -463,6 +652,33 @@ func (m Manager) probePublic(ctx context.Context, cfg config.Config, checkMarker return nil } +func (m Manager) continuityWindow(ctx context.Context, cfg config.Config, previousAddress string, checkMarkers bool) error { + steps := cfg.Deployment.ActivationWindowSecs * 4 + if steps < 1 { + steps = 1 + } + for step := 0; step < steps; step++ { + if err := m.probePublic(ctx, cfg, checkMarkers); err != nil { + return err + } + if previousAddress != "" { + if err := m.probeHealthReadiness(ctx, cfg, previousAddress); err != nil { + return fmt.Errorf("previous slot lost continuity: %w", err) + } + } + if step+1 < steps { + sleep := m.Sleep + if sleep == nil { + sleep = sleepContext + } + if err := sleep(ctx, 250*time.Millisecond); err != nil { + return err + } + } + } + return nil +} + func (m Manager) probe(ctx context.Context, cfg config.Config, address string, checks []config.Smoke) error { timeout := time.Duration(cfg.Deployment.CandidateTimeoutSecs) * time.Second for _, check := range checks { diff --git a/internal/deploy/deploy_test.go b/internal/deploy/deploy_test.go index 6fc5013..3d250c3 100644 --- a/internal/deploy/deploy_test.go +++ b/internal/deploy/deploy_test.go @@ -5,6 +5,7 @@ package deploy import ( "context" "errors" + "fmt" "os" "path/filepath" "strings" @@ -12,18 +13,23 @@ import ( "time" "gamertan.com/tend/internal/config" + "gamertan.com/tend/internal/eventlog" "gamertan.com/tend/internal/state" ) type fakeOperator struct { failReload bool + failReloadAt int + reloads int failRestartUnit string + failRestartOnce bool rejectMarkers bool active map[string]bool starts, stops, restarts []string probes []string publicProbes []string failPublic bool + failPublicAfter int candidateEnvironment map[string]string candidateFile string } @@ -31,6 +37,9 @@ type fakeOperator struct { func (f *fakeOperator) Restart(_ context.Context, unit string) error { f.restarts = append(f.restarts, unit) if unit == f.failRestartUnit { + if f.failRestartOnce { + f.failRestartUnit = "" + } return errors.New("injected restart failure") } f.active[unit] = true @@ -59,7 +68,7 @@ func (f *fakeOperator) StartCandidate(_ context.Context, unit, binary, environme } func (f *fakeOperator) ProbeURL(_ context.Context, value, contains string) error { f.publicProbes = append(f.publicProbes, value) - if f.failPublic { + if f.failPublic || (f.failPublicAfter > 0 && len(f.publicProbes) >= f.failPublicAfter) { return errors.New("injected public smoke failure") } if f.rejectMarkers && contains != "" { @@ -92,13 +101,44 @@ func TestPublicSmokeFailureRestoresBlueGreenHandlerAndSlot(t *testing.T) { if err != nil || target != old { t.Fatalf("green=%q err=%v", target, err) } - if _, err = os.Stat(cfg.Deployment.StateFile); !os.IsNotExist(err) { - t.Fatal("failed public smoke wrote state") + record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if err != nil { + t.Fatal(err) + } + if record.ActiveRelease != old || record.LastAttemptOutcome != "failed" || record.CandidateRelease != "" || record.LastAttemptRelease != fresh { + t.Fatalf("failed attempt state=%+v", record) + } +} + +func TestContinuityFailureRestoresBlueGreenRoute(t *testing.T) { + cfg, old, fresh := baseConfig(t, "blue_green") + handler := filepath.Join(cfg.Deployment.Root, "handler.caddy") + template := filepath.Join(cfg.Deployment.Root, "handler.template") + original := []byte("reverse_proxy 127.0.0.1:8090\n") + _ = os.WriteFile(handler, original, 0o644) + _ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644) + blue := filepath.Join(cfg.Deployment.Root, "slots", "blue") + green := filepath.Join(cfg.Deployment.Root, "slots", "green") + _ = replaceSymlink(blue, old) + _ = replaceSymlink(green, old) + cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}} + operator := &fakeOperator{active: map[string]bool{}, failPublicAfter: 3} + if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err == nil || !strings.Contains(err.Error(), "continuity") { + t.Fatalf("expected continuity failure, got %v", err) + } + body, _ := os.ReadFile(handler) + if string(body) != string(original) { + t.Fatalf("handler not restored: %q", body) + } + target, err := resolveReleaseLink(cfg.Deployment.Root, green) + if err != nil || target != old { + t.Fatalf("green=%q err=%v", target, err) } } func (f *fakeOperator) ValidateCaddy(context.Context, string) error { return nil } func (f *fakeOperator) ReloadCaddy(context.Context) error { - if f.failReload { + f.reloads++ + if f.failReload || (f.failReloadAt > 0 && f.reloads == f.failReloadAt) { return errors.New("injected reload failure") } return nil @@ -117,7 +157,7 @@ func baseConfig(t *testing.T, strategy string) (config.Config, string, string) { if err := os.MkdirAll(filepath.Join(root, "releases"), 0o755); err != nil { t.Fatal(err) } - old := filepath.Join(root, "releases", "legacy-old") + old := filepath.Join(root, "releases", "sha256-"+strings.Repeat("c", 64)) fresh := filepath.Join(root, "releases", "sha256-"+strings.Repeat("a", 64)) for _, dir := range []string{old, fresh} { if err := os.Mkdir(dir, 0o755); err != nil { @@ -127,11 +167,34 @@ func baseConfig(t *testing.T, strategy string) (config.Config, string, string) { t.Fatal(err) } } - cfg := config.Config{SchemaVersion: 2, Service: config.Service{Name: "example-site", AllowedHost: "example.test", EnvironmentFile: "/etc/tend/environment/example-site.env"}, Build: config.Build{Package: "./cmd/site", Binary: "app", Branch: "main"}, Deployment: config.Deployment{Strategy: strategy, Root: root, LockFile: filepath.Join(root, "deploy.lock"), StateFile: filepath.Join(root, "state.json"), HealthPath: "/healthz", ReadinessPath: "/readyz", CandidateTimeoutSecs: 2, Smoke: []config.Smoke{{Path: "/", Contains: "Example"}}, PublicSmoke: []config.PublicSmoke{{URL: "https://example.test/", Contains: "Example"}}}} + cfg := config.Config{SchemaVersion: 2, Service: config.Service{Name: "example-site", AllowedHost: "example.test", EnvironmentFile: "/etc/tend/environment/example-site.env"}, Build: config.Build{Package: "./cmd/site", Binary: "app", Branch: "main"}, Deployment: config.Deployment{Strategy: strategy, Root: root, LockFile: filepath.Join(root, "deploy.lock"), StateFile: filepath.Join(root, "state.json"), EventLog: filepath.Join(root, "deployment-events.jsonl"), HealthPath: "/healthz", ReadinessPath: "/readyz", CandidateTimeoutSecs: 2, ActivationWindowSecs: 1, Smoke: []config.Smoke{{Path: "/", Contains: "Example"}}, PublicSmoke: []config.PublicSmoke{{URL: "https://example.test/", Contains: "Example"}}}} return cfg, old, fresh } func manager(operator Operator, fresh string) Manager { - return Manager{Operator: operator, Now: func() time.Time { return time.Unix(100, 0).UTC() }, Prepare: func(config.Config, string, string, string) (string, error) { return fresh, nil }, Inspect: func(config.Config, string, string, string) error { return nil }} + return Manager{Operator: operator, Now: func() time.Time { return time.Unix(100, 0).UTC() }, Prepare: func(config.Config, string, string, string) (string, error) { return fresh, nil }, Inspect: func(config.Config, string, string, string) error { return nil }, ReadIdentity: func(string) (releaseIdentity, error) { + return releaseIdentity{Version: "v0.2.0-preview.1", Commit: strings.Repeat("b", 40)}, nil + }, OperationID: func() (string, error) { return strings.Repeat("d", 32), nil }, Sleep: func(context.Context, time.Duration) error { return nil }} +} + +func singletonSettings(t *testing.T, cfg config.Config, currentRelease string) (*config.Singleton, string) { + t.Helper() + handler := filepath.Join(cfg.Deployment.Root, "handler.caddy") + template := filepath.Join(cfg.Deployment.Root, "handler.template") + if err := os.WriteFile(handler, []byte("reverse_proxy 127.0.0.1:8092\n"), 0o640); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644); err != nil { + t.Fatal(err) + } + current := filepath.Join(cfg.Deployment.Root, "current") + if err := replaceSymlink(current, currentRelease); err != nil { + t.Fatal(err) + } + return &config.Singleton{ + Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN", + CurrentLink: current, PreviousLink: filepath.Join(cfg.Deployment.Root, "previous"), CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), + CaddyHandler: handler, CaddyHandlerTemplate: template, + }, handler } func TestBlueGreenActivationAndRollback(t *testing.T) { @@ -158,13 +221,28 @@ func TestBlueGreenActivationAndRollback(t *testing.T) { } operator := &fakeOperator{active: map[string]bool{"example-blue.service": true, "example-green.service": true}} m := manager(operator, fresh) - report, err := m.Deploy(context.Background(), cfg, Request{Activate: true}) + var events []eventlog.Event + m.AppendEvent = func(_ string, event eventlog.Event) error { + events = append(events, event) + return nil + } + report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}) if err != nil { t.Fatal(err) } if report.ActiveRelease != fresh || report.PreviousRelease != old { t.Fatalf("report=%+v", report) } + deployed, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if err != nil { + t.Fatal(err) + } + if deployed.DesiredRelease != fresh || deployed.CandidateRelease != "" || deployed.LastAttemptRelease != fresh || deployed.LastAttemptOutcome != "succeeded" { + t.Fatalf("deployment identity state=%+v", deployed) + } + if len(events) != 2 || events[0].Phase != "candidate" || events[0].Outcome != "running" || events[1].Phase != "activation" || events[1].Outcome != "succeeded" || events[0].OperationID != events[1].OperationID { + t.Fatalf("events=%+v", events) + } if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 { t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err) } @@ -180,6 +258,9 @@ func TestBlueGreenActivationAndRollback(t *testing.T) { if record.ActiveRelease != old || record.PreviousRelease != fresh { t.Fatalf("rollback=%+v", record) } + if len(events) != 4 || events[2].Phase != "rollback" || events[2].Outcome != "running" || events[3].Phase != "rollback" || events[3].Outcome != "succeeded" || events[2].OperationID != events[3].OperationID || events[2].ArtifactDigest != strings.Repeat("c", 64) { + t.Fatalf("rollback events=%+v", events) + } if len(operator.probes) != 4 { t.Fatalf("rollback probes=%#v", operator.probes) } @@ -190,6 +271,58 @@ func TestBlueGreenActivationAndRollback(t *testing.T) { } } +func TestDeploymentEvidenceCanNeverBlockActivationOrRollback(t *testing.T) { + cfg, old, fresh := baseConfig(t, "singleton_candidate") + cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old) + m := manager(&fakeOperator{active: map[string]bool{}}, fresh) + appendCalls := 0 + m.AppendEvent = func(string, eventlog.Event) error { + appendCalls++ + return errors.New("injected event failure") + } + report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}) + if err != nil { + t.Fatal(err) + } + if report.EventWarnings != 2 || report.ActiveRelease != fresh { + t.Fatalf("report=%+v", report) + } + if _, err := m.Rollback(context.Background(), cfg); err != nil { + t.Fatal(err) + } + record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if err != nil || record.ActiveRelease != old || appendCalls != 4 { + t.Fatalf("record=%+v appends=%d err=%v", record, appendCalls, err) + } +} + +func TestDeploymentEvidenceIdentityAndEntropyAreBestEffort(t *testing.T) { + for _, test := range []struct { + name string + damage func(*Manager) + }{ + {"identity", func(manager *Manager) { + manager.ReadIdentity = func(string) (releaseIdentity, error) { + return releaseIdentity{}, errors.New("injected identity failure") + } + }}, + {"entropy", func(manager *Manager) { + manager.OperationID = func() (string, error) { return "", errors.New("injected entropy failure") } + }}, + } { + t.Run(test.name, func(t *testing.T) { + cfg, old, fresh := baseConfig(t, "singleton_candidate") + cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old) + manager := manager(&fakeOperator{active: map[string]bool{}}, fresh) + test.damage(&manager) + report, err := manager.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}) + if err != nil || report.EventWarnings != 1 || report.ActiveRelease != fresh { + t.Fatalf("report=%+v err=%v", report, err) + } + }) + } +} + func TestBlueGreenCaddyFailureRestoresHandlerAndSlot(t *testing.T) { cfg, old, fresh := baseConfig(t, "blue_green") handler := filepath.Join(cfg.Deployment.Root, "handler.caddy") @@ -215,37 +348,47 @@ func TestBlueGreenCaddyFailureRestoresHandlerAndSlot(t *testing.T) { if err != nil || target != old { t.Fatalf("green=%q err=%v", target, err) } - if _, err := os.Stat(cfg.Deployment.StateFile); !os.IsNotExist(err) { - t.Fatal("failed activation wrote state") + failed, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if err != nil { + t.Fatal(err) + } + if failed.ActiveRelease != old || failed.LastAttemptOutcome != "failed" || failed.CandidateRelease != "" { + t.Fatalf("failed state=%+v", failed) } } func TestSingletonRestartFailureRestoresPointers(t *testing.T) { cfg, old, fresh := baseConfig(t, "singleton_candidate") - previous := filepath.Join(cfg.Deployment.Root, "previous") - current := filepath.Join(cfg.Deployment.Root, "current") - _ = replaceSymlink(current, old) - cfg.Deployment.Singleton = &config.Singleton{Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN", CurrentLink: current, PreviousLink: previous} - operator := &fakeOperator{active: map[string]bool{}, failRestartUnit: "example-site.service"} + settings, handler := singletonSettings(t, cfg, old) + cfg.Deployment.Singleton = settings + operator := &fakeOperator{active: map[string]bool{}, failRestartUnit: "example-site.service", failRestartOnce: true} m := manager(operator, fresh) if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil { t.Fatal("expected failure") } - target, err := resolveReleaseLink(cfg.Deployment.Root, current) + target, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink) if err != nil || target != old { t.Fatalf("current=%q err=%v", target, err) } - if _, err := os.Lstat(previous); !os.IsNotExist(err) { + if _, err := os.Lstat(cfg.Deployment.Singleton.PreviousLink); !os.IsNotExist(err) { t.Fatal("previous pointer was not restored") } + body, err := os.ReadFile(handler) + if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" { + t.Fatalf("handler=%q err=%v", body, err) + } + if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 { + t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err) + } + if operator.active["example-site-tend-candidate.service"] { + t.Fatal("candidate was not stopped after successful restoration") + } } -func TestStatePersistsOnlyAfterSuccessfulActivation(t *testing.T) { +func TestStateRecordsSuccessfulActivation(t *testing.T) { cfg, old, fresh := baseConfig(t, "singleton_candidate") - current := filepath.Join(cfg.Deployment.Root, "current") - previous := filepath.Join(cfg.Deployment.Root, "previous") - _ = replaceSymlink(current, old) - cfg.Deployment.Singleton = &config.Singleton{Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN", CurrentLink: current, PreviousLink: previous} + settings, handler := singletonSettings(t, cfg, old) + cfg.Deployment.Singleton = settings operator := &fakeOperator{active: map[string]bool{}} m := manager(operator, fresh) if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err != nil { @@ -261,4 +404,72 @@ func TestStatePersistsOnlyAfterSuccessfulActivation(t *testing.T) { if operator.candidateFile != cfg.Service.EnvironmentFile || len(operator.candidateEnvironment) != 1 || operator.candidateEnvironment["EXAMPLE_LISTEN"] != "127.0.0.1:18092" { t.Fatalf("candidate file=%q environment=%#v", operator.candidateFile, operator.candidateEnvironment) } + body, err := os.ReadFile(handler) + if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" { + t.Fatalf("handler=%q err=%v", body, err) + } + if operator.reloads != 2 || operator.active["example-site-tend-candidate.service"] { + t.Fatalf("reloads=%d active=%#v", operator.reloads, operator.active) + } +} + +func TestSingletonContinuityFailureRestoresHandlerPointersAndService(t *testing.T) { + cfg, old, fresh := baseConfig(t, "singleton_candidate") + settings, handler := singletonSettings(t, cfg, old) + cfg.Deployment.Singleton = settings + operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failPublicAfter: 4} + m := manager(operator, fresh) + if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil || !strings.Contains(err.Error(), "continuity") { + t.Fatalf("expected continuity failure, got %v", err) + } + current, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink) + if err != nil || current != old { + t.Fatalf("current=%q err=%v", current, err) + } + body, err := os.ReadFile(handler) + if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" { + t.Fatalf("handler=%q err=%v", body, err) + } + if operator.active["example-site-tend-candidate.service"] { + t.Fatal("candidate was not stopped after continuity restoration") + } +} + +func TestSingletonCaddyReloadFailuresRestorePriorRoute(t *testing.T) { + for _, reload := range []int{1, 2} { + t.Run(fmt.Sprintf("reload-%d", reload), func(t *testing.T) { + cfg, old, fresh := baseConfig(t, "singleton_candidate") + settings, handler := singletonSettings(t, cfg, old) + cfg.Deployment.Singleton = settings + operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failReloadAt: reload} + if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true}); err == nil || !strings.Contains(err.Error(), "Caddy reload failed") { + t.Fatalf("expected Caddy reload failure, got %v", err) + } + current, err := resolveReleaseLink(cfg.Deployment.Root, settings.CurrentLink) + if err != nil || current != old { + t.Fatalf("current=%q err=%v", current, err) + } + body, err := os.ReadFile(handler) + if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" { + t.Fatalf("handler=%q err=%v", body, err) + } + if operator.active["example-site-tend-candidate.service"] { + t.Fatal("candidate was not stopped after route restoration") + } + }) + } +} + +func TestSingletonIncompleteRecoveryKeepsProvenCandidateRunning(t *testing.T) { + cfg, old, fresh := baseConfig(t, "singleton_candidate") + settings, _ := singletonSettings(t, cfg, old) + cfg.Deployment.Singleton = settings + operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failReload: true} + _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true}) + if err == nil || !strings.Contains(err.Error(), "candidate remains routed for operator recovery") { + t.Fatalf("expected explicit incomplete recovery, got %v", err) + } + if !operator.active["example-site-tend-candidate.service"] { + t.Fatal("proven candidate was stopped despite incomplete route restoration") + } } diff --git a/internal/deploy/release.go b/internal/deploy/release.go index e3e7f4a..5222bd3 100644 --- a/internal/deploy/release.go +++ b/internal/deploy/release.go @@ -196,6 +196,15 @@ func extractArtifact(artifact, stage string) error { _ = out.Close() return err } + // OpenFile modes are filtered through the caller's umask. Tend is + // commonly invoked by a root account with umask 0077, while release + // binaries must remain executable by their dedicated service users. + // Reapply the validated, name-derived mode explicitly before the file + // becomes part of an immutable release. + if err := out.Chmod(mode); err != nil { + _ = out.Close() + return err + } if err := out.Sync(); err != nil { _ = out.Close() return err diff --git a/internal/deploy/release_mode_linux_test.go b/internal/deploy/release_mode_linux_test.go new file mode 100644 index 0000000..0ce20af --- /dev/null +++ b/internal/deploy/release_mode_linux_test.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//go:build linux + +package deploy + +import ( + "archive/tar" + "compress/gzip" + "os" + "path/filepath" + "syscall" + "testing" +) + +func TestExtractArtifactAppliesReleaseModesUnderRestrictiveUmask(t *testing.T) { + dir := t.TempDir() + artifact := filepath.Join(dir, "release.tar.gz") + file, err := os.OpenFile(artifact, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + gz := gzip.NewWriter(file) + tw := tar.NewWriter(gz) + entries := map[string][]byte{ + "bundle/app": []byte("executable"), + "bundle/BUILDINFO.json": []byte("{}"), + "bundle/RELEASE.json": []byte("{}"), + "bundle/SBOM.spdx.json": []byte("{}"), + "bundle/SHA256SUMS": []byte("checksums"), + } + for name, body := range entries { + if err := tw.WriteHeader(&tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0o600, Size: int64(len(body))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(body); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + + oldUmask := syscall.Umask(0o077) + t.Cleanup(func() { syscall.Umask(oldUmask) }) + stage := filepath.Join(dir, "stage") + if err := os.Mkdir(stage, 0o700); err != nil { + t.Fatal(err) + } + if err := extractArtifact(artifact, stage); err != nil { + t.Fatal(err) + } + + for name, want := range map[string]os.FileMode{ + "app": 0o755, + "BUILDINFO.json": 0o644, + "RELEASE.json": 0o644, + "SBOM.spdx.json": 0o644, + "SHA256SUMS": 0o644, + } { + info, err := os.Stat(filepath.Join(stage, name)) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != want { + t.Fatalf("%s mode=%#o want=%#o", name, got, want) + } + } +} diff --git a/internal/eventlog/eventlog.go b/internal/eventlog/eventlog.go new file mode 100644 index 0000000..5146dfe --- /dev/null +++ b/internal/eventlog/eventlog.go @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package eventlog + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "syscall" + "time" +) + +const Version = 1 + +var safeValue = regexp.MustCompile(`^[A-Za-z0-9._:/@+-]{1,256}$`) +var hexDigest = regexp.MustCompile(`^[0-9a-f]{64}$`) +var gitCommit = regexp.MustCompile(`^[0-9a-f]{40}$`) +var operationID = regexp.MustCompile(`^[0-9a-f]{32}$`) + +type Event struct { + Version int `json:"version"` + OperationID string `json:"operation_id"` + Service string `json:"service"` + ArtifactDigest string `json:"artifact_digest"` + Commit string `json:"commit"` + ReleaseVersion string `json:"release_version"` + Phase string `json:"phase"` + Slot string `json:"slot,omitempty"` + DurationMillis int64 `json:"duration_ms"` + Outcome string `json:"outcome"` + ObservedAt string `json:"observed_at"` +} + +func OperationID() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", errors.New("cryptographic randomness unavailable") + } + return hex.EncodeToString(b), nil +} + +func Append(path string, event Event) error { + if err := event.validate(); err != nil { + return err + } + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return errors.New("event log path must be absolute and clean") + } + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return fmt.Errorf("create event directory: %w", err) + } + if info, err := os.Lstat(path); err == nil { + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o022 != 0 { + return errors.New("event log must be a non-writable regular non-symlink file") + } + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect event log: %w", err) + } + b, err := json.Marshal(event) + if err != nil { + return fmt.Errorf("encode deployment event: %w", err) + } + if len(b) > 4096 { + return errors.New("deployment event exceeds bound") + } + b = append(b, '\n') + fd, err := syscall.Open(path, syscall.O_WRONLY|syscall.O_APPEND|syscall.O_CREAT|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0o640) + if err != nil { + return fmt.Errorf("open event log: %w", err) + } + file := os.NewFile(uintptr(fd), path) + if file == nil { + _ = syscall.Close(fd) + return errors.New("open event log file") + } + defer file.Close() + n, err := file.Write(b) + if err != nil || n != len(b) { + return errors.New("write complete deployment event") + } + if err := file.Sync(); err != nil { + return fmt.Errorf("sync deployment event: %w", err) + } + return nil +} + +func (e Event) validate() error { + if e.Version != Version || !operationID.MatchString(e.OperationID) { + return errors.New("deployment event identity is invalid") + } + if !hexDigest.MatchString(e.ArtifactDigest) || !gitCommit.MatchString(e.Commit) { + return errors.New("deployment event provenance is invalid") + } + for label, value := range map[string]string{"service": e.Service, "artifact_digest": e.ArtifactDigest, "commit": e.Commit, "release_version": e.ReleaseVersion, "phase": e.Phase, "outcome": e.Outcome} { + if !safeValue.MatchString(value) || strings.ContainsRune(value, '\x00') { + return fmt.Errorf("deployment event %s is invalid", label) + } + } + if e.Slot != "" && !safeValue.MatchString(e.Slot) { + return errors.New("deployment event slot is invalid") + } + if e.DurationMillis < 0 { + return errors.New("deployment event duration is invalid") + } + if _, err := time.Parse(time.RFC3339Nano, e.ObservedAt); err != nil { + return errors.New("deployment event timestamp is invalid") + } + return nil +} diff --git a/internal/eventlog/eventlog_test.go b/internal/eventlog/eventlog_test.go new file mode 100644 index 0000000..1351c6f --- /dev/null +++ b/internal/eventlog/eventlog_test.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package eventlog + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestAppendBoundedEvent(t *testing.T) { + path := filepath.Join(t.TempDir(), "events.jsonl") + id, err := OperationID() + if err != nil { + t.Fatal(err) + } + event := Event{Version: 1, OperationID: id, Service: "site", ArtifactDigest: strings.Repeat("a", 64), Commit: strings.Repeat("b", 40), ReleaseVersion: "v0.2.0-preview.1", Phase: "activation", Slot: "green", Outcome: "succeeded", ObservedAt: time.Now().UTC().Format(time.RFC3339Nano)} + if err := Append(path, event); err != nil { + t.Fatal(err) + } + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var decoded Event + if err := json.Unmarshal(b, &decoded); err != nil { + t.Fatal(err) + } + if decoded.OperationID != id || strings.Contains(string(b), "secret") { + t.Fatalf("unexpected event: %s", b) + } + if info, _ := os.Stat(path); info.Mode().Perm() != 0o640 { + t.Fatalf("mode=%04o", info.Mode().Perm()) + } +} + +func TestAppendRejectsUnboundedValuesAndSymlink(t *testing.T) { + id, _ := OperationID() + event := Event{Version: 1, OperationID: id, Service: "site\nsecret", ArtifactDigest: "digest", Commit: "commit", ReleaseVersion: "version", Phase: "activation", Outcome: "failed", ObservedAt: time.Now().UTC().Format(time.RFC3339Nano)} + if err := Append(filepath.Join(t.TempDir(), "events.jsonl"), event); err == nil { + t.Fatal("expected unsafe value rejection") + } + dir := t.TempDir() + target := filepath.Join(dir, "target") + if err := os.WriteFile(target, nil, 0o640); err != nil { + t.Fatal(err) + } + link := filepath.Join(dir, "events.jsonl") + if err := os.Symlink(target, link); err != nil { + t.Skip(err) + } + event.Service = "site" + if err := Append(link, event); err == nil { + t.Fatal("expected symlink rejection") + } +} diff --git a/internal/state/state.go b/internal/state/state.go index d314dd0..7332f85 100644 --- a/internal/state/state.go +++ b/internal/state/state.go @@ -17,13 +17,18 @@ import ( const SchemaVersion = 1 type Record struct { - SchemaVersion int `json:"schema_version"` - Strategy string `json:"strategy"` - ActiveSlot string `json:"active_slot"` - ActiveRelease string `json:"active_release"` - PreviousSlot string `json:"previous_slot,omitempty"` - PreviousRelease string `json:"previous_release,omitempty"` - UpdatedAt string `json:"updated_at"` + SchemaVersion int `json:"schema_version"` + Strategy string `json:"strategy"` + DesiredRelease string `json:"desired_release,omitempty"` + CandidateRelease string `json:"candidate_release,omitempty"` + ActiveSlot string `json:"active_slot"` + ActiveRelease string `json:"active_release"` + PreviousSlot string `json:"previous_slot,omitempty"` + PreviousRelease string `json:"previous_release,omitempty"` + LastAttemptRelease string `json:"last_attempt_release,omitempty"` + LastAttemptOutcome string `json:"last_attempt_outcome,omitempty"` + LastAttemptAt string `json:"last_attempt_at,omitempty"` + UpdatedAt string `json:"updated_at"` } func Load(path, root, strategy string) (Record, error) { @@ -68,6 +73,26 @@ func (r Record) Validate(root, strategy string) error { return fmt.Errorf("previous release: %w", err) } } + for label, release := range map[string]string{"desired release": r.DesiredRelease, "candidate release": r.CandidateRelease, "last attempt release": r.LastAttemptRelease} { + if release != "" { + if err := releaseBelow(root, release); err != nil { + return fmt.Errorf("%s: %w", label, err) + } + } + } + if r.LastAttemptOutcome != "" { + switch r.LastAttemptOutcome { + case "running", "succeeded", "failed", "rolled_back": + default: + return errors.New("last attempt outcome is invalid") + } + if _, err := time.Parse(time.RFC3339, r.LastAttemptAt); err != nil { + return errors.New("last attempt timestamp is invalid") + } + } + if r.LastAttemptOutcome == "running" && r.CandidateRelease == "" { + return errors.New("running attempt requires a candidate release") + } if strategy == "blue_green" && r.PreviousRelease != "" && r.PreviousSlot == r.ActiveSlot { return errors.New("previous slot must differ from active slot") } diff --git a/internal/state/state_test.go b/internal/state/state_test.go index f9be5ae..7a775da 100644 --- a/internal/state/state_test.go +++ b/internal/state/state_test.go @@ -16,7 +16,8 @@ func TestStoreLoadRoundTripAndRejectSymlink(t *testing.T) { t.Fatal(err) } path := filepath.Join(root, "state.json") - record := Record{SchemaVersion: 1, Strategy: "singleton_candidate", ActiveSlot: "singleton", ActiveRelease: release, UpdatedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)} + at := time.Unix(1, 0).UTC().Format(time.RFC3339) + record := Record{SchemaVersion: 1, Strategy: "singleton_candidate", DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, LastAttemptRelease: release, LastAttemptOutcome: "succeeded", LastAttemptAt: at, UpdatedAt: at} if err := Store(path, root, record); err != nil { t.Fatal(err) } @@ -24,8 +25,8 @@ func TestStoreLoadRoundTripAndRejectSymlink(t *testing.T) { if err != nil { t.Fatal(err) } - if loaded.ActiveRelease != release { - t.Fatalf("release=%q", loaded.ActiveRelease) + if loaded.ActiveRelease != release || loaded.DesiredRelease != release || loaded.LastAttemptOutcome != "succeeded" { + t.Fatalf("state=%+v", loaded) } if err := os.Remove(path); err != nil { t.Fatal(err) @@ -37,6 +38,15 @@ func TestStoreLoadRoundTripAndRejectSymlink(t *testing.T) { t.Fatal("expected symlink refusal") } } + +func TestRecordRequiresCandidateForRunningAttempt(t *testing.T) { + root := filepath.Join(t.TempDir(), "service") + release := filepath.Join(root, "releases", "sha256-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + record := Record{SchemaVersion: 1, Strategy: "singleton_candidate", DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, LastAttemptRelease: release, LastAttemptOutcome: "running", LastAttemptAt: time.Unix(1, 0).UTC().Format(time.RFC3339), UpdatedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)} + if err := record.Validate(root, "singleton_candidate"); err == nil { + t.Fatal("expected missing candidate rejection") + } +} func TestRecordRejectsReleaseOutsideRoot(t *testing.T) { record := Record{SchemaVersion: 1, Strategy: "singleton_candidate", ActiveSlot: "singleton", ActiveRelease: "/tmp/other/release", UpdatedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)} if err := record.Validate("/opt/example", "singleton_candidate"); err == nil { diff --git a/release/tend.json b/release/tend.json index e11f053..e3611be 100644 --- a/release/tend.json +++ b/release/tend.json @@ -14,9 +14,11 @@ "root": "/opt/tend-release-test", "lock_file": "/run/lock/tend-deploy.lock", "state_file": "/opt/tend-release-test/tend-state.json", + "event_log": "/opt/tend-release-test/deployment-events.jsonl", "health_path": "/healthz", "readiness_path": "/readyz", "candidate_timeout_seconds": 5, + "activation_window_seconds": 5, "smoke": [{ "path": "/", "contains": "Tend" }], "public_smoke": [{ "url": "https://example.test/", "contains": "Tend" }], "singleton": { @@ -25,7 +27,10 @@ "candidate_address": "127.0.0.1:19091", "listen_env": "TEND_RELEASE_TEST_LISTEN", "current_link": "/opt/tend-release-test/current", - "previous_link": "/opt/tend-release-test/previous" + "previous_link": "/opt/tend-release-test/previous", + "caddy_config": "/etc/caddy/Caddyfile", + "caddy_handler": "/etc/caddy/tend-release-test-handler.caddy", + "caddy_handler_template": "/etc/tend/caddy/tend-release-test.template" } } } diff --git a/scripts/check-public-tree.sh b/scripts/check-public-tree.sh new file mode 100755 index 0000000..51ca25f --- /dev/null +++ b/scripts/check-public-tree.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +set -euo pipefail +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +allow=$root/scripts/public-snapshot.allow +LC_ALL=C sort -c "$allow" +[[ $(LC_ALL=C sort "$allow" | uniq -d | wc -l) -eq 0 ]] +mapfile -t files <"$allow" +[[ ${#files[@]} -gt 0 ]] +for file in "${files[@]}"; do + [[ -n $file && $file != /* && $file != *..* && $file != .gitea/* && $file != .github/* ]] + git -C "$root" cat-file -e "HEAD:$file" +done +work=$(mktemp -d) +trap 'rm -rf -- "$work"' EXIT +mkdir -m 0700 "$work/tree" +git -C "$root" archive HEAD -- "${files[@]}" | tar -xf - -C "$work/tree" +test ! -e "$work/tree/.git" +test ! -e "$work/tree/.gitea" +test ! -e "$work/tree/.github" +private_pattern='/home/[[:alnum:]_.-]+/|BEGIN (RSA|OPENSSH|EC) PRIVATE KEY|gitea[-_]api[[:alnum:]_.-]*token' +if (cd "$work/tree" && rg -n --hidden --glob '!scripts/export-public.sh' --glob '!scripts/check-public-tree.sh' "$private_pattern" .); then + echo "private material found in public tree" >&2 + exit 1 +fi +(cd "$work/tree" && ./scripts/check-licenses.sh) +(cd "$work/tree" && GOWORK=off go test -count=1 ./...) +(cd "$work/tree" && GOWORK=off go vet ./...) +(cd "$work/tree" && GOWORK=off CGO_ENABLED=0 go build -buildvcs=false -mod=readonly -trimpath -o "$work/tend" ./cmd/tend) +echo "public tree compiles independently" diff --git a/scripts/export-public.sh b/scripts/export-public.sh index ee143c7..150a4b1 100755 --- a/scripts/export-public.sh +++ b/scripts/export-public.sh @@ -44,7 +44,8 @@ done < <(find "$stage" -type f -print | LC_ALL=C sort) epoch=$(git -C "$root" show -s --format=%ct "$commit") printf '{"schema_version":1,"source_commit":"%s","source_tree":"%s","source_date_epoch":%s,"file_count":%d}\n' "$commit" "$tree" "$epoch" "${#files[@]}" >"$stage/PUBLIC-SNAPSHOT.json" (cd "$stage" && sha256sum PUBLIC-SNAPSHOT.json >PUBLIC-SNAPSHOT.sha256) -if (cd "$stage" && rg -n --hidden --glob '!.git/**' --glob '!PUBLIC-SNAPSHOT.json' --glob '!scripts/export-public.sh' '/home/cole|BEGIN (RSA|OPENSSH|EC) PRIVATE KEY|gitea-api\.token' .); then +private_pattern='/home/[[:alnum:]_.-]+/|BEGIN (RSA|OPENSSH|EC) PRIVATE KEY|gitea[-_]api[[:alnum:]_.-]*token' +if (cd "$stage" && rg -n --hidden --glob '!.git/**' --glob '!PUBLIC-SNAPSHOT.json' --glob '!scripts/export-public.sh' "$private_pattern" .); then echo "private material found" >&2 exit 1 fi diff --git a/scripts/public-snapshot.allow b/scripts/public-snapshot.allow index 7e4342b..2cb7cee 100644 --- a/scripts/public-snapshot.allow +++ b/scripts/public-snapshot.allow @@ -8,6 +8,7 @@ SECURITY.md cmd/tend/main.go docs/ARCHITECTURE.md docs/DOGFOOD_EVIDENCE.md +docs/DOGFOOD_FRICTION.md docs/PUBLIC_SNAPSHOT.md docs/SCHEMA_V2_MIGRATION.md docs/THREAT_MODEL.md @@ -19,6 +20,8 @@ examples/blue-green/example-site@.service examples/blue-green/tend.json examples/local/.env.example examples/server/authorized_keys.example +examples/server/caddy/docs-site.template +examples/server/caddy/example-site.template examples/server/environment/docs-site.env.example examples/server/environment/example-site.env.example examples/server/example-singleton.service @@ -28,6 +31,7 @@ examples/server/services/example-site.json examples/server/slots/example-site-blue.env examples/server/slots/example-site-green.env examples/server/tend-receive.sudoers +examples/singleton/caddy-handler.template examples/singleton/tend.json go.mod internal/config/config.go @@ -42,7 +46,10 @@ internal/deploy/operator_test.go internal/deploy/ownership_linux.go internal/deploy/ownership_other.go internal/deploy/release.go +internal/deploy/release_mode_linux_test.go internal/deploy/release_test.go +internal/eventlog/eventlog.go +internal/eventlog/eventlog_test.go internal/packager/packager.go internal/packager/packager_test.go internal/process/run.go @@ -62,6 +69,7 @@ internal/version/version.go internal/version/version_test.go release/tend.json scripts/check-licenses.sh +scripts/check-public-tree.sh scripts/export-public.sh scripts/public-snapshot.allow scripts/test-public-snapshot.sh diff --git a/scripts/verify.sh b/scripts/verify.sh index 877cce5..89ce5ae 100755 --- a/scripts/verify.sh +++ b/scripts/verify.sh @@ -4,6 +4,7 @@ set -euo pipefail root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) cd "$root" ./scripts/check-licenses.sh +./scripts/check-public-tree.sh go test -count=1 ./... go test -race -count=1 ./... go vet ./...