From bf56dbce0f71280e81fd574c615d839a8cf5b2ad Mon Sep 17 00:00:00 2001 From: Cole Speelman Date: Tue, 18 Aug 2026 21:42:33 -0400 Subject: [PATCH] docs: publish Tend Compose continuity evidence Export the reviewed allowlisted snapshot from private source commit 07c1655921f21ee5e4fc4d85639d199e8867b17d. This records the Docker Compose activation, schema-compatible rollback, and stateful migration resource findings from Observatory Preview 19 dogfooding. AI-Assisted: OpenAI Codex Signed-off-by: Cole Speelman --- .gitattributes | 5 + .gitignore | 5 + COPYRIGHT | 4 + LICENSE | 662 +++++++++++ PUBLIC-SNAPSHOT.json | 1 + PUBLIC-SNAPSHOT.sha256 | 1 + README.md | 102 ++ RELEASE.md | 50 + SECURITY.md | 17 + cmd/tend/main.go | 392 +++++++ docs/ARCHITECTURE.md | 94 ++ docs/DOGFOOD_EVIDENCE.md | 297 +++++ docs/DOGFOOD_FRICTION.md | 656 +++++++++++ docs/PUBLIC_SNAPSHOT.md | 19 + docs/SCHEMA_V2_MIGRATION.md | 44 + docs/THREAT_MODEL.md | 55 + docs/WALKTHROUGH.md | 98 ++ examples/LICENSE | 12 + examples/README.md | 41 + examples/blue-green/caddy-handler.template | 2 + examples/blue-green/example-site@.service | 18 + examples/blue-green/tend.json | 26 + examples/local/.env.example | 3 + examples/server/authorized_keys.example | 2 + examples/server/caddy/docs-site.template | 2 + examples/server/caddy/example-site.template | 2 + .../server/environment/docs-site.env.example | 3 + .../environment/example-site.env.example | 2 + examples/server/example-singleton.service | 19 + examples/server/receive-policy.json | 16 + examples/server/services/docs-site.json | 29 + examples/server/services/example-site.json | 29 + examples/server/slots/example-site-blue.env | 2 + examples/server/slots/example-site-green.env | 2 + examples/server/tend-receive.sudoers | 3 + examples/singleton/caddy-handler.template | 2 + examples/singleton/tend.json | 29 + go.mod | 3 + internal/config/config.go | 348 ++++++ internal/config/config_test.go | 90 ++ internal/deploy/deploy.go | 1023 +++++++++++++++++ internal/deploy/deploy_test.go | 591 ++++++++++ internal/deploy/lock_linux.go | 32 + internal/deploy/lock_linux_test.go | 30 + internal/deploy/lock_other.go | 14 + internal/deploy/operator.go | 125 ++ internal/deploy/operator_test.go | 49 + internal/deploy/ownership_linux.go | 38 + internal/deploy/ownership_other.go | 17 + internal/deploy/reconcile.go | 270 +++++ internal/deploy/reconcile_test.go | 182 +++ internal/deploy/release.go | 318 +++++ internal/deploy/release_mode_linux_test.go | 75 ++ internal/deploy/release_test.go | 41 + internal/eventlog/eventlog.go | 115 ++ internal/eventlog/eventlog_test.go | 59 + internal/packager/packager.go | 399 +++++++ internal/packager/packager_test.go | 77 ++ internal/process/run.go | 72 ++ internal/provenance/git.go | 80 ++ internal/provenance/git_test.go | 52 + internal/serverpolicy/ownership_linux.go | 14 + internal/serverpolicy/ownership_other.go | 8 + internal/serverpolicy/policy.go | 300 +++++ internal/serverpolicy/policy_test.go | 59 + internal/state/lease.go | 156 +++ internal/state/lease_test.go | 89 ++ internal/state/state.go | 172 +++ internal/state/state_test.go | 55 + internal/transport/protocol.go | 123 ++ internal/transport/protocol_test.go | 81 ++ internal/transport/push.go | 153 +++ internal/transport/push_test.go | 81 ++ internal/transport/receive.go | 48 + internal/version/version.go | 82 ++ internal/version/version_test.go | 33 + release/tend.json | 36 + scripts/check-licenses.sh | 11 + scripts/check-public-tree.sh | 30 + scripts/export-public.sh | 54 + scripts/public-snapshot.allow | 81 ++ scripts/test-public-snapshot.sh | 18 + scripts/verify.sh | 25 + 83 files changed, 8555 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 COPYRIGHT create mode 100644 LICENSE create mode 100644 PUBLIC-SNAPSHOT.json create mode 100644 PUBLIC-SNAPSHOT.sha256 create mode 100644 README.md create mode 100644 RELEASE.md create mode 100644 SECURITY.md create mode 100644 cmd/tend/main.go create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/DOGFOOD_EVIDENCE.md create mode 100644 docs/DOGFOOD_FRICTION.md create mode 100644 docs/PUBLIC_SNAPSHOT.md create mode 100644 docs/SCHEMA_V2_MIGRATION.md create mode 100644 docs/THREAT_MODEL.md create mode 100644 docs/WALKTHROUGH.md create mode 100644 examples/LICENSE create mode 100644 examples/README.md create mode 100644 examples/blue-green/caddy-handler.template create mode 100644 examples/blue-green/example-site@.service create mode 100644 examples/blue-green/tend.json create mode 100644 examples/local/.env.example create mode 100644 examples/server/authorized_keys.example create mode 100644 examples/server/caddy/docs-site.template create mode 100644 examples/server/caddy/example-site.template create mode 100644 examples/server/environment/docs-site.env.example create mode 100644 examples/server/environment/example-site.env.example create mode 100644 examples/server/example-singleton.service create mode 100644 examples/server/receive-policy.json create mode 100644 examples/server/services/docs-site.json create mode 100644 examples/server/services/example-site.json create mode 100644 examples/server/slots/example-site-blue.env create mode 100644 examples/server/slots/example-site-green.env create mode 100644 examples/server/tend-receive.sudoers create mode 100644 examples/singleton/caddy-handler.template create mode 100644 examples/singleton/tend.json create mode 100644 go.mod create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go create mode 100644 internal/deploy/deploy.go create mode 100644 internal/deploy/deploy_test.go create mode 100644 internal/deploy/lock_linux.go create mode 100644 internal/deploy/lock_linux_test.go create mode 100644 internal/deploy/lock_other.go create mode 100644 internal/deploy/operator.go create mode 100644 internal/deploy/operator_test.go create mode 100644 internal/deploy/ownership_linux.go create mode 100644 internal/deploy/ownership_other.go create mode 100644 internal/deploy/reconcile.go create mode 100644 internal/deploy/reconcile_test.go create mode 100644 internal/deploy/release.go create mode 100644 internal/deploy/release_mode_linux_test.go create mode 100644 internal/deploy/release_test.go create mode 100644 internal/eventlog/eventlog.go create mode 100644 internal/eventlog/eventlog_test.go create mode 100644 internal/packager/packager.go create mode 100644 internal/packager/packager_test.go create mode 100644 internal/process/run.go create mode 100644 internal/provenance/git.go create mode 100644 internal/provenance/git_test.go create mode 100644 internal/serverpolicy/ownership_linux.go create mode 100644 internal/serverpolicy/ownership_other.go create mode 100644 internal/serverpolicy/policy.go create mode 100644 internal/serverpolicy/policy_test.go create mode 100644 internal/state/lease.go create mode 100644 internal/state/lease_test.go create mode 100644 internal/state/state.go create mode 100644 internal/state/state_test.go create mode 100644 internal/transport/protocol.go create mode 100644 internal/transport/protocol_test.go create mode 100644 internal/transport/push.go create mode 100644 internal/transport/push_test.go create mode 100644 internal/transport/receive.go create mode 100644 internal/version/version.go create mode 100644 internal/version/version_test.go create mode 100644 release/tend.json create mode 100755 scripts/check-licenses.sh create mode 100755 scripts/check-public-tree.sh create mode 100755 scripts/export-public.sh create mode 100644 scripts/public-snapshot.allow create mode 100755 scripts/test-public-snapshot.sh create mode 100755 scripts/verify.sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d5b18af --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +* text=auto eol=lf +*.go text eol=lf +*.json text eol=lf +*.md text eol=lf +*.sh text eol=lf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6ebbaa9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +/bin/ +/dist/ +*.tmp +.env.local +*.env.local diff --git a/COPYRIGHT b/COPYRIGHT new file mode 100644 index 0000000..a67600f --- /dev/null +++ b/COPYRIGHT @@ -0,0 +1,4 @@ +Copyright 2026 Cole Speelman + +Except where a file or subtree says otherwise, this repository is licensed +under AGPL-3.0-only. Reusable examples under examples/ are licensed 0BSD. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2beb9e1 --- /dev/null +++ b/LICENSE @@ -0,0 +1,662 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. + diff --git a/PUBLIC-SNAPSHOT.json b/PUBLIC-SNAPSHOT.json new file mode 100644 index 0000000..c18fff7 --- /dev/null +++ b/PUBLIC-SNAPSHOT.json @@ -0,0 +1 @@ +{"schema_version":1,"source_commit":"07c1655921f21ee5e4fc4d85639d199e8867b17d","source_tree":"0a27c7c0d2fb8954e326044d06826cf7e1a37818","source_date_epoch":1787103648,"file_count":81} diff --git a/PUBLIC-SNAPSHOT.sha256 b/PUBLIC-SNAPSHOT.sha256 new file mode 100644 index 0000000..098e955 --- /dev/null +++ b/PUBLIC-SNAPSHOT.sha256 @@ -0,0 +1 @@ +a38ba7894dede461d7aec3f1b07c5c6f65728f2971b973fc81c469c32667d38e PUBLIC-SNAPSHOT.json diff --git a/README.md b/README.md new file mode 100644 index 0000000..f90add4 --- /dev/null +++ b/README.md @@ -0,0 +1,102 @@ +# Gamertan Tend + +Tend is an opinionated release and deployment tool for small Go services on +Linux, systemd, and Caddy. It packages a clean pushed commit, records exact +build provenance, activates a health-checked candidate, and keeps rollback +state explicit. + +The v0.2 development line supports two quiet deployment shapes: + +- Caddy-switched blue/green services. +- A singleton service with an isolated transient candidate check. + +It does not manage databases, migrations, containers, Kubernetes, or arbitrary +shell hooks. Application-specific data activation remains application-specific. + +## Commands + +```text +tend check --config tend.json +tend package --config tend.json --version v0.2.0-preview.2 --out dist +tend inspect --config tend.json --artifact FILE --sha256 HEX --approve-sha256 HEX +tend push --target tend-deploy@host --known-hosts FILE --service NAME --artifact FILE --sha256 HEX --approve-sha256 HEX +tend receive --policy /etc/tend/receive-policy.json +tend check-server --policy /etc/tend/receive-policy.json +tend deploy --config /etc/tend/services/example-site.json --artifact FILE --sha256 HEX --approve-sha256 HEX +tend status --config /etc/tend/services/example-site.json +tend reconcile --config /etc/tend/services/example-site.json --json +tend rollback --config /etc/tend/services/example-site.json +tend prune --config /etc/tend/services/example-site.json --keep 3 [--apply] +``` + +`inspect` performs the complete archive, checksum, approval, manifest, SBOM, +and binary-identity validation without staging or activating a release. +`reconcile` is also read-only: it compares Tend's journal with conventional +release symlinks, systemd units, the installed release identity, and the Caddy +handler. It reports drift and retained-candidate residue without silently +"repairing" a service. + +`push` transfers one approved artifact through a pinned OpenSSH connection. A +forced, no-shell receiver maps the service name to one root-owned configuration; +it accepts no remote path, environment value, URL, or shell fragment. Production +hosts receive binaries and evidence, never source or Go dependencies. + +Schema 2 keeps all services under `/etc/tend/services/`, references a distinct +root-owned `0600` environment file for each service, and serializes activation +through `/run/lock/tend-deploy.lock`. Builds and transfers remain parallel; +only the short Caddy/service activation phase is host-wide. Tend is still a +single command, not a daemon. + +Each service also keeps a bounded JSONL deployment-event stream and explicit +desired, candidate, active, previous, and last-attempt release identities. The +stream contains only fixed provenance and lifecycle fields; Observatory may +ingest it later, but an event-write failure never blocks deployment or rollback. +After Caddy reload, Tend repeatedly probes the configured canonical HTTPS +origins for the activation window. Blue/green deployments simultaneously keep +checking the previous slot, restoring the prior handler and inactive-slot state +if routed traffic or continuity fails. + +Singleton activation and rollback candidates use an operation-scoped systemd +unit and persist a bounded lease containing the operation, release, unit, +address, and start time in a separate adjacent file. Conventional deployment +state remains schema 1 so a retained older binary can still read active and +previous release identities; operators must reconcile before downgrading while +a lease exists. `tend +reconcile --json` compares that lease with unit activity, release pointers, and +the exact Caddy handler file without changing any of them. It deliberately does +not stop or clear a retained candidate: that process may still be the only +healthy route after an interrupted recovery. + +Dry-run and digest approval are intentional friction. See the +[schema-2 migration guide](docs/SCHEMA_V2_MIGRATION.md) and the +[two-service walkthrough](docs/WALKTHROUGH.md). + +Run `./scripts/verify.sh` on Linux. That required release lane exercises tests, +the race detector, vet, deterministic builds, schema-2 examples, and the +dependency-free module graph. Tend supports Linux hosts with systemd and Caddy; +WSL may be used as a Linux development environment, but native Windows is not a +supported execution, deployment, or release-gate platform. + +The v0.1 public preview and immutable `v0.2.0-preview.1` each completed +maintenance releases and explicit rollback/reactivation for both Gamertan and +the Sandwich Hime website using one reviewed candidate. See the dated +[dogfood evidence](docs/DOGFOOD_EVIDENCE.md) for exact scope and limitations. + +The additive `v0.2.0-preview.2` candidate keeps that transport and activation +contract while adding bounded deployment-event JSONL, routed-origin continuity, +rollback annotations, and the operational findings recorded through real +dogfooding. Preview 1 remains unchanged. Preview 2 will not be tagged until one +identical binary has deployed, rolled back, and reactivated Gamertan, the +Sandwich Hime website, and Gamertan Observatory. +Operational friction discovered while applying the same contract to new +services is tracked separately in the +[dogfood friction ledger](docs/DOGFOOD_FRICTION.md). The ledger preserves the +fail-closed behavior and records candidate product improvements instead of +normalizing application-specific deployment workarounds. +The canonical public repository begins with a sanitized root snapshot rather +than the private development history. + +## Licensing + +Tend and its release machinery are AGPL-3.0-only. Reusable example +configuration and service templates under `examples/` are 0BSD. diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..6425f9f --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,50 @@ +# Preview release policy + +`v0.1.0-preview.2` may be published only after one identical candidate Tend +binary has successfully completed maintenance releases for Gamertan and the +Sandwich Hime website, including injected-failure restoration and explicit +rollback proof. The August 14, 2026 campaign met that gate; the scoped, +sanitized record is in `docs/DOGFOOD_EVIDENCE.md`. + +Before a preview tag: + +1. Run `./scripts/verify.sh` from a clean pushed private-development commit. +2. Review dependency, license, race, filesystem, archive, and rollback evidence. +3. Export the exact allowlisted public tree into a new root commit. +4. Verify canonical Gitea and GitHub discovery trees are byte-identical. +5. Sign the canonical Gitea tag and attach checksums and an SPDX SBOM. +6. Verify a fresh public checkout before advertising installation. + +The release tag and attached candidate must be built from the final reviewed +source commit. Documentation-only changes after the recorded campaign require +one final identical-candidate maintenance pass before tagging. + +`v0.2.0-preview.1` is an immutable, additive release line. It requires schema 2, +the restricted `push`/`receive` transport, root-owned environment-file +references, host-wide activation serialization, and HTTPS public-origin smoke. +Its exact released source remains unchanged. + +`v0.2.0-preview.2` adds bounded deployment-event JSONL, routed-origin +activation continuity, rollback annotations, and the reviewed operational +friction record. It must preserve every Preview 1 security and release gate. +The exact same Preview 2 binary must deploy, roll back, and reactivate +Gamertan, the Sandwich Hime website, and Gamertan Observatory before the tag is +created. EQL is not part of this generic gate; its SQLite/catalog publication +needs a dedicated adapter rather than arbitrary hooks. + +The operation-scoped lease compatibility candidate at private commit +`789976ce766fd457ca10762540135e4e0e74cfe3` has completed the Gamertan and +Sandwich Hime deployment, rollback, reactivation, and failure gates. It has not +completed the Observatory gate because the live service now uses Docker +Compose, outside the released strategy contract. Do not create the Preview 2 +tag from this candidate until that explicit topology decision is resolved. + +`v0.1.0-preview.1` is immutable but withdrawn: its source and module checksums +are valid, while a fresh `go install` reports the development identity because +the CLI did not yet adopt the tagged module version from Go build information. +Preview 2 adds that identity path and its regression tests; preview 1 is never +retagged or rewritten. + +The canonical public origin is `ssh://git@gitea.speelman.ca:2222/gamertan/tend.git`. +GitHub is a read-only discovery snapshot. Private development history is not +published or merged into either public history. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..a746e01 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,17 @@ +# Security policy + +Report suspected vulnerabilities privately to `security@sandwichhime.com`. +Please include the affected Tend version, configuration shape, reproduction, +and expected impact. Do not include production credentials or private logs. + +Tend treats reviewed source, pinned toolchains, root-owned server policy and +configuration, handwritten Caddy templates, and operators as trusted. It treats +requested service names, protocol frames, artifact paths and bytes, archives, +filesystem state, process output, HTTP responses, and deployment targets as +adversarial inputs. It never evaluates configuration as shell code. + +The preview is not a sandbox and does not make an untrusted repository safe to +build. Run `tend package` only for reviewed source. Production configuration, +receive policy, host keys, identities, and `0600` environment files stay outside +repositories. A secret value appearing in a Tend report, artifact, state file, +or process argument is a security defect and should be reported. diff --git a/cmd/tend/main.go b/cmd/tend/main.go new file mode 100644 index 0000000..fd9bd43 --- /dev/null +++ b/cmd/tend/main.go @@ -0,0 +1,392 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "os/signal" + "path/filepath" + "runtime" + "syscall" + "time" + + "gamertan.com/tend/internal/config" + "gamertan.com/tend/internal/deploy" + "gamertan.com/tend/internal/packager" + "gamertan.com/tend/internal/process" + "gamertan.com/tend/internal/serverpolicy" + "gamertan.com/tend/internal/transport" + "gamertan.com/tend/internal/version" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "tend:", err) + os.Exit(1) + } +} +func run() error { + if len(os.Args) < 2 { + return usage() + } + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + switch os.Args[1] { + case "check": + return checkCommand(os.Args[2:]) + case "package": + return packageCommand(ctx, os.Args[2:]) + case "deploy": + return deployCommand(ctx, os.Args[2:]) + case "inspect": + return inspectCommand(ctx, os.Args[2:]) + case "push": + return pushCommand(ctx, os.Args[2:]) + case "receive": + return receiveCommand(ctx, os.Args[2:]) + case "check-server": + return checkServerCommand(os.Args[2:]) + case "status": + return statusCommand(ctx, os.Args[2:]) + case "reconcile": + return reconcileCommand(ctx, os.Args[2:]) + case "rollback": + return rollbackCommand(ctx, os.Args[2:]) + case "prune": + return pruneCommand(os.Args[2:]) + case "version": + return writeJSON(map[string]string{"version": version.Version, "commit": version.Commit, "date": version.Date, "go": runtime.Version()}) + default: + return usage() + } +} +func usage() error { + return errors.New("usage: tend [options]") +} + +func checkCommand(args []string) error { + set := flag.NewFlagSet("check", flag.ContinueOnError) + set.SetOutput(os.Stderr) + path := set.String("config", "", "absolute path to tend.json") + if err := set.Parse(args); err != nil { + return err + } + if set.NArg() != 0 { + return errors.New("check accepts no positional arguments") + } + cfg, absolute, err := loadConfig(*path) + if err != nil { + return err + } + return writeJSON(map[string]any{"valid": true, "config": absolute, "schema_version": cfg.SchemaVersion, "service": cfg.Service.Name, "strategy": cfg.Deployment.Strategy}) +} +func packageCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("package", flag.ContinueOnError) + set.SetOutput(os.Stderr) + path := set.String("config", "", "path to tend.json") + source := set.String("source", ".", "clean source checkout") + out := set.String("out", "dist", "artifact directory") + releaseVersion := set.String("version", "", "vX.Y.Z-preview.N") + if err := set.Parse(args); err != nil { + return err + } + if set.NArg() != 0 { + return errors.New("package accepts no positional arguments") + } + cfg, _, err := loadConfig(*path) + if err != nil { + return err + } + sourceAbs, err := filepath.Abs(*source) + if err != nil { + return err + } + outAbs, err := filepath.Abs(*out) + if err != nil { + return err + } + result, err := packager.Package(ctx, process.ExecRunner{}, cfg, sourceAbs, outAbs, *releaseVersion) + if err != nil { + return err + } + return writeJSON(result) +} +func deployCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("deploy", flag.ContinueOnError) + set.SetOutput(os.Stderr) + path := set.String("config", "", "absolute target configuration") + artifact := set.String("artifact", "", "release archive") + sha := set.String("sha256", "", "expected artifact SHA-256") + approved := set.String("approve-sha256", "", "separately reviewed artifact SHA-256") + activate := set.Bool("activate", false, "perform activation after validation") + if err := set.Parse(args); err != nil { + return err + } + if set.NArg() != 0 { + return errors.New("deploy accepts no positional arguments") + } + cfg, absolute, err := loadConfig(*path) + if err != nil { + return err + } + if *activate { + if err := requireMutationAuthority(absolute, cfg); err != nil { + return err + } + } + artifactAbs, err := filepath.Abs(*artifact) + if err != nil { + return err + } + report, err := newManager().Deploy(ctx, cfg, deploy.Request{Artifact: artifactAbs, SHA256: *sha, ApprovedSHA256: *approved, Activate: *activate}) + if err != nil { + return err + } + return writeJSON(report) +} + +func inspectCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("inspect", flag.ContinueOnError) + set.SetOutput(os.Stderr) + path := set.String("config", "", "target configuration") + artifact := set.String("artifact", "", "release archive") + sha := set.String("sha256", "", "expected artifact SHA-256") + approved := set.String("approve-sha256", "", "separately reviewed artifact SHA-256") + if err := set.Parse(args); err != nil { + return err + } + if set.NArg() != 0 { + return errors.New("inspect accepts no positional arguments") + } + cfg, _, err := loadConfig(*path) + if err != nil { + return err + } + artifactAbs, err := filepath.Abs(*artifact) + if err != nil { + return err + } + report, err := newManager().Deploy(ctx, cfg, deploy.Request{ + Artifact: artifactAbs, + SHA256: *sha, + ApprovedSHA256: *approved, + Activate: false, + }) + if err != nil { + return err + } + return writeJSON(report) +} + +func pushCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("push", flag.ContinueOnError) + set.SetOutput(os.Stderr) + target := set.String("target", "", "dedicated tend-deploy user and host") + port := set.Int("port", 22, "pinned SSH port") + knownHosts := set.String("known-hosts", "", "absolute pinned known-hosts file") + identity := set.String("identity", "", "optional absolute private key") + service := set.String("service", "", "allowed service name") + artifact := set.String("artifact", "", "immutable release archive") + sha := set.String("sha256", "", "expected artifact SHA-256") + approved := set.String("approve-sha256", "", "separately reviewed artifact SHA-256") + activate := set.Bool("activate", false, "activate after server validation") + if err := set.Parse(args); err != nil { + return err + } + if set.NArg() != 0 { + return errors.New("push accepts no positional arguments") + } + artifactAbs, err := filepath.Abs(*artifact) + if err != nil { + return err + } + result, err := transport.Push(ctx, transport.ExecSSHRunner{}, transport.PushOptions{Target: *target, Port: *port, KnownHosts: *knownHosts, Identity: *identity, Service: *service, Artifact: artifactAbs, SHA256: *sha, ApprovedSHA256: *approved, Activate: *activate}) + if err != nil { + return err + } + return writeJSON(result) +} + +func receiveCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("receive", flag.ContinueOnError) + set.SetOutput(os.Stderr) + policyPath := set.String("policy", "/etc/tend/receive-policy.json", "root-owned receive policy") + if err := set.Parse(args); err != nil { + return err + } + if set.NArg() != 0 { + return errors.New("receive accepts no positional arguments") + } + if runtime.GOOS != "linux" || os.Geteuid() != 0 { + return errors.New("receive requires Linux root execution") + } + if os.Getenv("SSH_ORIGINAL_COMMAND") != transport.Protocol { + return errors.New("receive refused unexpected SSH command") + } + policy, err := serverpolicy.Load(*policyPath) + if err != nil { + return err + } + report, err := transport.Receive(ctx, os.Stdin, policy, newManager()) + if err != nil { + return err + } + return writeJSON(report) +} + +func checkServerCommand(args []string) error { + set := flag.NewFlagSet("check-server", flag.ContinueOnError) + set.SetOutput(os.Stderr) + policyPath := set.String("policy", "/etc/tend/receive-policy.json", "root-owned receive policy") + if err := set.Parse(args); err != nil { + return err + } + if set.NArg() != 0 { + return errors.New("check-server accepts no positional arguments") + } + if runtime.GOOS != "linux" || os.Geteuid() != 0 { + return errors.New("check-server requires Linux root execution") + } + policy, err := serverpolicy.Load(*policyPath) + if err != nil { + return err + } + services, err := policy.CheckFiles() + if err != nil { + return err + } + names := make([]string, 0, len(services)) + for _, service := range services { + names = append(names, service.Name) + } + return writeJSON(map[string]any{"valid": true, "schema_version": policy.SchemaVersion, "services": names}) +} +func statusCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("status", flag.ContinueOnError) + set.SetOutput(os.Stderr) + path := set.String("config", "", "absolute target configuration") + if err := set.Parse(args); err != nil { + return err + } + if set.NArg() != 0 { + return errors.New("status accepts no positional arguments") + } + cfg, _, err := loadConfig(*path) + if err != nil { + return err + } + result, err := newManager().Status(ctx, cfg) + if err != nil { + return err + } + return writeJSON(result) +} +func reconcileCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("reconcile", flag.ContinueOnError) + set.SetOutput(os.Stderr) + path := set.String("config", "", "absolute target configuration") + _ = set.Bool("json", false, "emit the reconciliation report as JSON") + if err := set.Parse(args); err != nil { + return err + } + if set.NArg() != 0 { + return errors.New("reconcile accepts no positional arguments") + } + cfg, _, err := loadConfig(*path) + if err != nil { + return err + } + report, err := newManager().Reconcile(ctx, cfg) + if err != nil { + return err + } + return writeJSON(report) +} +func rollbackCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("rollback", flag.ContinueOnError) + set.SetOutput(os.Stderr) + path := set.String("config", "", "absolute target configuration") + activate := set.Bool("activate", false, "perform the recorded rollback") + if err := set.Parse(args); err != nil { + return err + } + if set.NArg() != 0 { + return errors.New("rollback accepts no positional arguments") + } + cfg, absolute, err := loadConfig(*path) + if err != nil { + return err + } + if !*activate { + return writeJSON(map[string]any{"validated": true, "mutation": "none", "message": "repeat with --activate to switch to the recorded previous release"}) + } + if err := requireMutationAuthority(absolute, cfg); err != nil { + return err + } + record, err := newManager().Rollback(ctx, cfg) + if err != nil { + return err + } + return writeJSON(record) +} +func pruneCommand(args []string) error { + set := flag.NewFlagSet("prune", flag.ContinueOnError) + set.SetOutput(os.Stderr) + path := set.String("config", "", "absolute target configuration") + keep := set.Int("keep", 3, "number of active, previous, and recent releases to retain") + apply := set.Bool("apply", false, "remove the listed releases") + if err := set.Parse(args); err != nil { + return err + } + if set.NArg() != 0 { + return errors.New("prune accepts no positional arguments") + } + cfg, absolute, err := loadConfig(*path) + if err != nil { + return err + } + if *apply { + if err := requireMutationAuthority(absolute, cfg); err != nil { + return err + } + } + removed, err := newManager().Prune(cfg, *keep, *apply) + if err != nil { + return err + } + return writeJSON(map[string]any{"apply": *apply, "releases": removed}) +} +func loadConfig(path string) (config.Config, string, error) { + if path == "" { + return config.Config{}, "", errors.New("--config is required") + } + absolute, err := filepath.Abs(path) + if err != nil { + return config.Config{}, "", err + } + cfg, err := config.Load(absolute) + return cfg, absolute, err +} +func newManager() deploy.Manager { + return deploy.NewManager(deploy.SystemOperator{Runner: process.ExecRunner{}, Timeout: 5 * time.Second}) +} +func requireMutationAuthority(configPath string, cfg config.Config) error { + if runtime.GOOS != "linux" { + return errors.New("deployment mutations require Linux") + } + if os.Geteuid() != 0 { + return errors.New("deployment mutations require root") + } + return serverpolicy.CheckConfig(configPath, cfg) +} +func writeJSON(value any) error { + encoder := json.NewEncoder(os.Stdout) + encoder.SetEscapeHTML(false) + encoder.SetIndent("", " ") + return encoder.Encode(value) +} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..09af8b3 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,94 @@ +# Architecture + +Tend has three deliberately separate boundaries. + +## Package boundary + +`tend package` accepts only a clean checkout whose `HEAD` exactly matches the +configured pushed branch. It rejects Go module replacements and unversioned +dependencies, builds the configured main package twice with `GOWORK=off`, and +requires byte-identical output. The resulting archive contains only: + +- the service binary; +- `RELEASE.json`; +- `BUILDINFO.json`; +- `SBOM.spdx.json`; and +- `SHA256SUMS`. + +The external archive digest is the release identity used by deployment. + +## Transport boundary + +`tend push` sends one bounded protocol frame to a dedicated account through a +pinned OpenSSH host key. It uses an argument vector, disables config files, +forwarding, PTYs, local commands, and proxy commands, and requests exactly +`tend-receive-v1`. The account's forced command invokes only the root-owned +receiver. The receiver's root-owned policy maps an allowlisted service name to +one exact configuration path and size ceiling. + +The target receives no source, repository credential, Go cache, dependency, or +arbitrary command. The artifact digest must equal both the produced digest and +the separately supplied approved digest before it is staged. + +## Target-host boundary + +Schema-2 configurations live below `/etc/tend/services/`. Secrets live only in +separate `/etc/tend/environment/*.env` files that are root-owned, non-symlink, +and mode `0600`. Configurations contain the path, never the values. Candidate +and installed units read the same file; Tend overrides only the candidate's +loopback listen address. + +Dry-run validation extracts into a temporary directory and does not touch the +release tree. Activation acquires the shared lock, installs a content-addressed +release, verifies all embedded checksums and manifest fields, and starts the +candidate without evaluating configuration as shell code. + +## Activation boundary + +Blue/green mode points the inactive slot at the new release, restarts and +probes it, atomically replaces one imported Caddy handler, validates the full +Caddy configuration, reloads Caddy, and records the prior active slot. + +Singleton mode starts the target release in a hardened transient systemd unit on a +separate loopback address and probes it. The unit is named from the service and +a fresh bounded operation ID; a separate strict candidate-lease file binds that +unit to the candidate release, address, and start time without changing the +schema-1 deployment-state contract used by retained recovery binaries. Tend +then validates and atomically +routes the imported Caddy handler to that candidate. While the candidate serves +the canonical origin, Tend changes the current release pointer, restarts and +probes the installed fixed-address unit, validates Caddy again, and routes back +to it. The candidate remains healthy through the activation window and stops +only after the handoff succeeds. + +After the local post-activation probes, Tend also checks configured HTTPS public +origins throughout a bounded activation window. Blue/green mode also probes the +previous slot, and singleton mode probes the handoff candidate, for health and +readiness throughout that window. Any failure before success is recorded +restores the previously observed Caddy bytes and/or release pointers. State +records desired, candidate, active, previous, and last-attempt +release identities, including failed attempts without claiming they became +active. Rollback is a separate explicit command +over the recorded state. It rechecks local health/readiness and public reachability, +but deliberately does not apply a future release's content marker to an older +release whose routes may differ. Pruning preserves both active and previous releases. + +`tend reconcile` is a read-only observation boundary. It compares persisted +state, systemd activity, current/previous release pointers, and exact handler +file bytes. It does not claim to inspect Caddy's currently loaded in-memory +configuration, and it never clears or stops a candidate. The report makes +retained, inactive, ambiguous, and settled states legible before a future +explicit recovery operation is approved. + +## Evidence boundary + +Every attempted activation and explicit rollback emits bounded, versioned JSONL +events with an operation ID, service, approved artifact digest, source commit, +release version, phase, slot, elapsed duration, and outcome. Values are +validated rather than copied from command output. The log contains no +environment values, arbitrary process output, HTTP bodies, or secret paths. +Release-identity, event-file, and downstream observability failures are +deliberately best effort and cannot control Tend's deployment or rollback +result. A fresh operation identity is operationally required for a singleton +candidate unit; entropy failure therefore stops either forward activation or +rollback before a candidate process or traffic change. diff --git a/docs/DOGFOOD_EVIDENCE.md b/docs/DOGFOOD_EVIDENCE.md new file mode 100644 index 0000000..6876da5 --- /dev/null +++ b/docs/DOGFOOD_EVIDENCE.md @@ -0,0 +1,297 @@ +# Preview dogfood evidence + +This is maintainer-run operational evidence, not an independent audit or a +general reliability claim. It preserves each dated campaign and its limitations +instead of rewriting earlier observations as though later fixes had already +existed. + +## Operation-scoped lease compatibility campaign — August 18, 2026 + +The candidate-lease compatibility fix completed trusted verification, exact +candidate reproduction, live activation, rollback, reactivation, and a bounded +failure injection on the production Linux/systemd/Caddy host. Conventional +deployment state remained schema 1 throughout. Singleton operation identity +used the separate adjacent lease introduced by this candidate. + +### Assessed Tend candidate + +- Private implementation source commit: + `789976ce766fd457ca10762540135e4e0e74cfe3`. +- Version: `v0.2.0-preview.2`. +- Linux/amd64 binary SHA-256: + `d0109bf037e493c7a046defe37818c3c1811806360b99c9a0910213665bd95c5`. +- Release-candidate archive SHA-256: + `2c225e0b9dd0be2d36fda62ab8d0b52cd9b28f9336c60dfb5edf3e715f450f95`. +- Toolchain: Go 1.26.6, `CGO_ENABLED=0`, `-trimpath`. +- Trusted verification run 366 and release-candidate run 369 passed for the + exact commit. The release workflow built the archive twice with identical + bytes and published checksums, build metadata, and an SPDX SBOM. +- The forge upload action reported a finalized 3.18 MB artifact, but both + documented artifact-list endpoints returned an empty result. A clean + independent clone therefore built the package twice and reproduced the + workflow's exact archive digest before deployment. This is recorded as an + artifact-publication limitation, not described as a successful consumer + download. +- The previous installed Tend binary remained retained by checksum before the + candidate replaced the active tool. Server policy validation then passed. + +### Live maintenance and failure evidence + +Gamertan activated archive +`c17b4db1a4e2fd5406b392ec72195b947272097e4d1bfcec9d700d0889c3a4c6`, +rolled back to its recorded previous archive, and reactivated the first archive. +Both blue/green units remained active with zero restarts. Reconciliation was +settled after every transition. + +The Sandwich Hime website performed the same sequence with archive +`d418f93ced3307fa788f5d2209b5f09f16bceabec4a9698a9beec6a05e439f35`. +Every successful forward and rollback operation removed its operation-scoped +candidate lease and transient unit. A final intentional local-smoke failure +asked the otherwise valid candidate for an impossible marker. The candidate +failed before Caddy or release-pointer mutation, stopped, removed its lease, +recorded the failed attempt without changing the active release, and reconciled +as settled and consistent. + +After the campaign, Gamertan, its news and case-study indexes, Sandwich Hime, +its documentation and tutorial, the Gamertan-mounted Sandwich route, and EQL +health all returned HTTP 200. Caddy, both Gamertan slots, and the installed +Sandwich Hime service were active with zero restarts and no warning-or-higher +journal entries during the campaign. + +Observatory is not claimed by this candidate campaign. Its current dogfood +deployment is a Docker Compose singleton, which remains outside Tend's +versioned systemd/Caddy strategies. The earlier three-service campaign remains +valid for its assessed candidate, but the current commit must not be tagged +until Observatory either returns to a supported topology or a separately +reviewed Compose strategy completes the same activation, rollback, and failure +gates. + +## Final Preview 2 code-candidate campaign — August 18, 2026 + +The final v0.2 Preview 2 implementation candidate completed two explicit +rollback-and-reactivation cycles for Gamertan, the Sandwich Hime website, and +Gamertan Observatory on the production Linux/systemd/Caddy host. The candidate +routed singleton traffic to the already-proven transient process while the +fixed-address installed unit restarted, then restored the canonical upstream +only after loopback and public-origin validation. + +### Assessed Tend candidate + +- Private implementation source commit: + `1fd3b9904c46e817c244196dd5d5a90921ca81a2`. +- Version: `v0.2.0-preview.2`. +- Linux/amd64 binary SHA-256: + `adb4753d4e865775d50d618c999f10dfac9a0de945ccfd9d0b8f2e80d65f4597`. +- Gitea release-candidate archive SHA-256: + `2bdfee2168cb16883d524cf9703adfa6ed5bc2bf44fbbb896993acc5fe6969ec`. +- Toolchain: Go 1.26.6, `CGO_ENABLED=0`, `-trimpath`. +- Trusted verification run 286 and release-candidate run 287 passed. The + archive's checksums, SPDX SBOM, embedded version, commit, clean VCS state, + target, and Go build information were independently rechecked before host + installation. + +### Application maintenance artifacts + +- Gamertan archive SHA-256: + `4700b075640b8b2fb5c17e0e02cf8d96ee67ceee10fe76d108c8b411983a88aa`. +- Sandwich Hime website archive SHA-256: + `46b4da41cf6703fb8818e7d25e3a9c13e57cf700f5608b1888c4adb3352e4d38`. +- Observatory preview 12 archive SHA-256: + `9ef0ddd8ec25d8fb75d6a6887e3ba874df7ebba16d3254f6250fcc646f4fd7f4`. + +Every service ended with the intended current release active and the older +release retained as the explicit rollback target. Gamertan returned to its +green slot; both singleton services returned to their fixed addresses. +Candidate ports and the shared lock were free afterward. Caddy and all five +installed application units were active with zero restarts and no failed +units. + +### Continuity and deployment evidence + +A seven-minute workstation probe sampled the Gamertan origin, both Sandwich +Hime origins, Observatory, and EQL health 1,606 times each throughout the +campaign. It observed no HTTP failure status. One simultaneous client-side +disconnect affected all five destinations during a reload. A controlled +90-second replay therefore observed the same validated no-content-change Caddy +reload from both the workstation and an independent Linux host. The Linux host +recorded 450 successful responses and zero failures for every origin; the +workstation alone repeated one common-mode URL transport error across every +destination. Caddy retained the same PID with zero restarts and continued +serving unrelated requests. The common-mode workstation event is recorded as +an observer-path limitation, not server downtime. + +The authoritative deployment-event files finished at 18 Gamertan events, 14 +Sandwich Hime events, and 16 Observatory events. Observatory's agent cursor for +each stream exactly equalled the corresponding file size, proving complete +consumption. The agent and applications reported no warning-or-higher journal +entries during the campaign. Representative public routes returned HTTP 200, +and the EQL origin remained healthy. + +### Findings closed before this campaign + +Two earlier pre-activation artifacts exposed a mode-restoration defect under a +hardened root umask. They failed before route or pointer mutation. Tend now +reapplies validated archive modes explicitly and tests extraction under umask +`0077`. + +The older singleton strategy briefly exposed an unavailable fixed upstream and +produced transient Observatory-agent `502`s. The final candidate's routed +handoff removed that failure in repeated production activation and rollback. + +This evidence update changes VCS build metadata but not deployment logic. The +release policy therefore still requires one last maintenance pass with the +exact evidence-bearing candidate before the signed public tag is created. + +## Restricted multi-service campaign — August 16, 2026 + +Tend's v0.2 implementation candidate completed the same maintenance contract +through the restricted transport for two independently configured services on +one Linux/systemd/Caddy host. + +### Assessed Tend implementation + +- Implementation source commit: + `840b77da708bbcd87a3203fb6a1f99b2984b8667`. +- Linux/amd64 candidate binary SHA-256: + `b3961315288871fa6085bf3b75c784a825a88f2bc3dd0694ad5d6c590eddf895`. +- Candidate archive SHA-256: + `948bad5271dca08b9445c387c5aea7f58f22add6b47b9071e1801a756aa71259`. +- Toolchain: Go 1.26.6, `CGO_ENABLED=0`, `-trimpath`. +- Trusted Gitea verification run 119 and release-candidate run 120 passed for + the exact implementation commit. CI and an independent, network-disabled + build each produced the same archive digest twice. +- The installed receiver accepted only a pinned Ed25519 host key, a dedicated + forced-command deployment key, the exact `tend-receive-v1` protocol, two + allowlisted service names, and separately repeated artifact digests. An + attempted arbitrary SSH command was refused. +- Production source, Go caches, repository credentials, secret values, remote + paths, and shell commands did not cross the transport boundary. + +Adding this evidence changes VCS build metadata but not executable logic. The +signed preview tag and attached release evidence therefore identify the final +evidence-bearing candidate and its required last maintenance pass. + +### Gamertan blue-green campaign + +- Application source commit: + `a7e54047d3dc11671824b6ecc8ed698a9dd04421`. +- Preview 27 artifact SHA-256: + `edadef3a97c089771e2b6bb7dadd58928762284878f76f23b112c61ae282f17f`. +- The artifact was built twice, byte-identically, in the pinned Go 1.26.6 + image with networking disabled and the audited cached module graph. +- The inactive slot passed health, readiness, page-marker, Caddy validation, + public-origin, Sandwich Hime mount, and EQL continuity checks before traffic + moved. +- Explicit rollback restored preview 26; its readiness and public boundary + passed; preview 27 was then reactivated through the recorded state. +- Sandwich Hime's release pointer and service remained unchanged throughout. + +### Sandwich Hime singleton campaign + +- Application source commit: + `435880c6751b773b6c5ee3ae6833d26e8eb7c0df`. +- Preview 30 artifact SHA-256: + `bee8d5bcde2c3c2e8bb96d5909a19889fc7f0be9390046c243fe20c9ff2ca44b`. +- The artifact was built twice, byte-identically, in the same pinned, + network-disabled Go 1.26.6 environment. +- A transient DynamicUser candidate passed health, readiness, tutorial marker, + canonical-origin, and Gamertan-mounted checks before the singleton pointer + and installed service changed. The candidate port was released afterward. +- Explicit rollback restored preview 28; its readiness and public boundary + passed; preview 30 was then reactivated through the recorded state. +- Both Gamertan slots, its selected Caddy upstream, and EQL remained healthy. + +### Findings resolved by dogfood + +The first restricted transfer stopped before artifact validation because +`sudo` removed `SSH_ORIGINAL_COMMAND`. The forced account still refused the +request; no service state changed. The 0BSD sudoers template now preserves only +that one server-supplied variable, while the receiver requires its exact +protocol value and rejects every other command. + +The first singleton candidate stopped before pointer mutation because the +shared environment file's live listen value overrode the candidate address. +The site remained on its former release and retained the same process. Tend now +rejects singleton shared environment files containing the configured listen +key; installed units own the non-secret live address and Tend supplies only the +transient candidate address. + +Final verification found both service states valid, every installed unit and +Caddy active with zero restart failures, the candidate port closed, no warning +or error entries after the successful campaign, and representative Gamertan, +Sandwich Hime, mounted, and EQL routes returning HTTP 200. + +## Assessed candidate + +- Source commit: `306d085e518cb4fe7b20a66d1e2ceb171e54ebdc`. +- Linux/amd64 candidate binary SHA-256: + `a9d53e286317d5acad9c0c321dc8d6240efee1e992714a892aba5be7c190dffc`. +- Candidate archive SHA-256: + `52639d16cd55b1dfe7c4ce63d4523676a6e8fe7cf57ef25787938c4070f49bd3`. +- Toolchain: Go 1.26.6, `CGO_ENABLED=0`, `-trimpath`. +- Two fresh packages from the clean pushed commit were byte-identical. +- Tests, race tests, vet, license checks, public-snapshot isolation, and + `govulncheck v1.1.4` passed. The vulnerability scan reported no known + reachable vulnerabilities with the August 14, 2026 database. +- Trusted Gitea verification completed successfully for the exact commit. + +The release tag and attached assets must still identify their own exact source +commit and digests. Any code change after this campaign requires the dogfood +sequence to be repeated. + +A fresh-install check after the first immutable source tag found that the CLI +reported its development identity instead of the tagged module version. No +deployment logic or artifact content was ambiguous, but the distribution +identity was not acceptable. Preview 1 remains immutable and withdrawn; +Preview 2 adds Go build-information version selection and repeats the release +gates rather than retagging old content. + +## Sandwich Hime website + +The singleton-candidate strategy packaged and activated website preview 25: + +- application source commit: + `0429e3f0160aa4fd4d262bc5857bc232c2149cb8`; +- artifact SHA-256: + `25025a05bb1aa6689f5f0779064b24a7c8193ff93c1395a2b3bd342588c3926a`; +- application toolchain: Go 1.26.6; +- isolated transient candidate passed health, readiness, and application + smoke checks before the singleton pointer changed; +- explicit rollback returned to preview 24, and the exact approved preview 25 + artifact was then reactivated; +- canonical, documentation, news, `llms.txt`, and Gamertan-mounted routes + returned HTTP 200 after reactivation. + +## Gamertan + +The blue-green strategy packaged and activated Gamertan preview 8: + +- application source commit: + `3acfa6a8e66ca3827c840d1fe9bc0b51c69c0a45`; +- artifact SHA-256: + `76db7a9a6c496c204f653dc5e42c935272159320cc344b6a4db6374535c696ad`; +- application toolchain: Go 1.26.6; +- the inactive slot passed health, readiness, and page-marker checks before a + validated atomic Caddy handler replacement; +- the handler retained `root:caddy` ownership, mode `0640`, and Sandwich Hime + routing precedence; +- explicit rollback restored preview 7, and the exact approved preview 8 + artifact was then reactivated; +- homepage, project pages, news, feed, discovery files, the Sandwich Hime + mount, and EQL Helper continuity returned HTTP 200 after reactivation. + +## Finding resolved during the campaign + +An earlier rollback attempt stopped safely before changing traffic because it +applied the new release's content markers to an older release whose route set +was different. Tend now uses the full configured smoke suite for new +deployments and health/readiness checks for an already-recorded rollback +target. A regression test requires that separation. The fixed candidate then +completed both live rollback sequences. + +## Boundaries + +The campaign covered one Linux/systemd/Caddy host and two small Go services. +It did not cover databases, migrations, containers, Kubernetes, hostile root, +or EQL Helper's application-specific catalog activation. Artifact transport +remains an application-owned, host-key-verified step outside Tend v0.1. diff --git a/docs/DOGFOOD_FRICTION.md b/docs/DOGFOOD_FRICTION.md new file mode 100644 index 0000000..13a9040 --- /dev/null +++ b/docs/DOGFOOD_FRICTION.md @@ -0,0 +1,656 @@ +# Dogfood friction ledger + +This document records friction observed through August 19, 2026 while using +Tend for real maintenance releases. Friction is evidence about the product: it +should become either a clearer contract, safer automation, or an explicit +non-goal. It must not become application-specific shell lore. + +The findings below are maintainer observations, not implemented promises. Tend +continues to fail closed while an option is being designed. In particular, +there is no `--skip-remote` packaging escape hatch, no arbitrary deployment +hook, and no relaxation of the host-wide activation lock. + +## Design standard + +A Tend workflow should make the secure path the short, documented path: + +- source, artifact, approval, configuration, and active-release identities are + explicit and inspectable; +- credentials are narrowly scoped, briefly available, and absent from + artifacts and logs; +- application validation is declarative, bounded, and cannot become a shell; +- first installation, maintenance, activation, rollback, and pruning are + distinct operations; +- every mutation has a recorded prior state and a tested restoration path; +- Tend-owned state is an evidence journal and reconciliation aid, not an + exclusive claim over an otherwise conventional systemd/Caddy service; +- tool-owned transient resources are leased, inspectable, resumable, and safe + to reconcile without requiring operators to understand internal names; +- unrelated services may build and prepare candidates concurrently, while the + shared Caddy activation boundary remains serialized. + +## Observed findings + +| ID | Status | Finding | Current safe behavior | Candidate direction | +| --- | --- | --- | --- | --- | +| F-01 | Workflow mitigation | CI packaging needs proof that the exact commit was pushed. | Packaging verifies the configured remote and fails if Git cannot authenticate. The trusted workflow retains only its job-scoped read-only checkout credential. | Make the proof credential or an authenticated source attestation an explicit Tend input and evidence boundary. | +| F-02 | Open | Application-owned unit and configuration changes are outside the binary activation transaction. | Operators stage, validate, back up, and restore those files separately. | Add an allowlisted configuration transaction or record and validate exact configuration digests. | +| F-03 | Open | A singleton candidate may not reproduce the installed unit's arguments and application configuration. | It receives the production environment file and an isolated listen override; activation still fails closed. | Add a bounded application preflight and explicit, validated candidate invocation. | +| F-04 | Partially resolved in development | Artifact production, review, approval, transfer, and activation require several identity checks. | v0.2 standardizes the evidence bundle and restricted transfer; `tend inspect` now exposes the complete non-mutating artifact validation as a named workflow. | Preserve explicit approval while adding digest-bound publication receipts. | +| F-05 | Partially resolved | Tend maintenance assumes an existing adopted service and current release. | `check-server` validates a prepared host, while first installation remains a separately reviewed operator procedure. | Define an explicit `install` or `adopt` transaction rather than silently treating bootstrap as maintenance. | +| F-06 | Partially resolved | Binary rollback can be unsafe when service configuration has changed incompatibly. | v0.2 records desired, candidate, active, previous, and last-attempt releases, but external configuration compatibility is operator-owned. | Bind non-secret configuration identities and preflight results to release state. | +| F-07 | Open | `GOPROXY=off` does not prove that every module metadata lookup is available locally. | Packaging stops before artifact creation when Go cannot resolve the complete pinned module graph. | Separate checksum-verified dependency resolution from a network-disabled, cache-completeness-checked build stage. | +| F-08 | Partially resolved in development | Packaging an otherwise clean pushed commit from a linked Git worktree fails Go's required VCS-status stamp. | Tend now detects a linked worktree before remote proof and build work and gives an exact standalone-clone instruction. | Assess provenance-preserving linked-worktree support without weakening `-buildvcs=true`. | +| F-09 | Resolved in Preview 2 | Restarting a fixed-address singleton briefly exposed Caddy to an unavailable upstream. | Tend now routes the imported handler to the candidate, restarts and proves the installed unit behind that handoff, then routes back while the candidate remains healthy through the activation window. | Preserve the production regression campaign and failure-injection matrix. | +| F-10 | Resolved in Preview 2 | A hardened root umask could make an extracted application binary executable only by root. | Tend reapplies every validated archive mode explicitly; extraction is tested under umask `0077`. | Preserve the mode and non-root candidate tests. | +| F-11 | Evidence practice | A single external observer can report common-mode client or network errors as apparent multi-service downtime. | Production campaigns retain server state and use an independent observer before classifying a continuity failure. | Standardize multi-vantage continuity evidence without making an observability dependency part of deployment authority. | +| F-12 | Partially resolved in development | A deliberately retained singleton handoff candidate could occupy Tend's fixed transient-unit name and block the next valid activation. | Candidates now have operation-scoped unit identities and bounded persisted leases; `reconcile --json` reports release identity, unit, pointer, and handler-file facts without mutation. | Add an explicit, idempotent `resume`/repair operation only after its route-identity and failure-injection model is proven. | +| F-13 | Open | A configured public-origin probe may resolve to a different deployment after a DNS or edge migration. | Operators separately verify local routed-origin identity and external DNS topology before activation. | Bind probes to expected release and edge identities; classify topology drift instead of treating a body marker as deployment proof. | +| F-14 | Open | An application can bind durable data identity to an absolute path that a binary-only candidate never exercises. | Preserve the production path and run application-owned validation before traffic moves. | Add a bounded, explicitly read-only application preflight and record which configuration/data identities it assessed. | +| F-15 | Product direction | Requiring Tend-specific residue and metadata to be perfect can make the facilitator feel like an exclusive deployment owner. | Conventional systemd/Caddy recovery remains authoritative and every manual intervention is captured as evidence. | Make state append-only and migratable, infer observed state safely, and keep Tend removable without making the service obscure or undeployable. | +| F-16 | Open | A reusable Docker network alias can identify both the live service and a retained handoff candidate, making the routed target ambiguous during activation. | Pin the temporary Caddy handoff to the exact, observed candidate address and restore the reviewed application handler after activation. | Allocate operation-scoped network identities and reject any candidate or handler target that resolves to more than one container or release identity. | +| F-17 | Open | A CI job can report a successful artifact upload while the forge artifact API exposes no retrievable artifact to the approval/deployment client. | Reproduce and verify the exact pushed commit locally, then compare two packages byte-for-byte before approving the digest; never guess an artifact URL. | Define a digest-bound artifact handoff with an independently readable receipt and fail the workflow unless the approval client can retrieve and inspect the exact uploaded bytes. | +| F-18 | Open | Tend's current strategies do not model a Docker Compose singleton, so a manual Observatory replacement briefly exposed an unavailable upstream. | The exact image, data backup, rollback image, and health probes were preserved, but Compose replaced the only live container and an external observer saw a bounded `502`/`503` window. | Design an explicit Compose strategy with operation-scoped candidates, unambiguous routing, durable-data preflight, activation continuity, and automatic rollback; do not add arbitrary container hooks to the systemd strategies. | +| F-19 | Open | A successful application schema migration can make the previous binary unreadable even when its image and service definition are intact. | Preserve verified database backups and prefer completing the proven forward activation; do not execute a binary-only rollback after a forward-only migration. | Make applications declare schema compatibility and an explicit data restoration or forward-recovery plan before Tend offers automatic rollback. | +| F-20 | Open | A stateful migration can require bounded temporary scratch space that a stateless binary candidate never exercises. | Run the exact candidate against a copied production data set under its proposed resource limits; record the reviewed scratch budget in the service definition. | Bind resource/configuration digests to the candidate and add an application-owned stateful preflight contract without arbitrary hooks. | + +## F-01: pushed-commit proof in CI + +### Observation + +During an Observatory release-candidate run, verification intentionally checked +out source without persisted credentials. Tend later attempted to prove that +`HEAD` existed on the configured private Gitea branch and Git could not +authenticate. Packaging stopped before producing an artifact. This was the +correct fail-closed outcome, but the credential lifecycle was not obvious from +the workflow contract. + +The trusted workflow uses a job-scoped read-only checkout credential so Tend +can perform the independent remote proof. It is not a long-lived repository or +deployment credential, and it is removed by checkout cleanup at the end of the +job. Production credentials remain unavailable to the build job. + +### Options to assess + +1. Support an explicit read-only Git credential file or credential helper for + package-time proof. Never accept a token in arguments, configuration, + manifests, artifacts, or logs. +2. Accept an authenticated CI source-attestation document binding repository, + branch, commit, tree, workflow identity, and event identity. Define which + CI issuers are trusted and preserve the attestation with release evidence. +3. Accept a signed, pre-verified source bundle whose identity and policy can be + checked without network access. + +Do not add a generic skip flag. An unavailable proof must remain a packaging +failure unless an equally strong proof mode was selected explicitly. + +## F-02: application configuration is not binary activation + +### Observation + +Tend can transact an immutable binary, release pointers, a systemd restart, a +validated Caddy handler, health checks, and rollback state. It does not +currently transact application configuration, systemd unit changes, credential +bindings, or server-local secret files. A first deployment that changes these +files therefore needs a separate backup, validation, installation, and +restoration procedure. + +This boundary is safe but easy to overlook: restoring the previous binary does +not restore an incompatible unit or application configuration. + +### Options to assess + +1. Add strict managed-file entries with exact source and destination paths, + content digest, owner, group, mode, and validation type. Permit only regular + files beneath configured roots; reject symlinks and unknown destinations. +2. Add a separate `tend configure` transaction that backs up, atomically + replaces, validates, and restores supported systemd/Caddy/application files. +3. Keep configuration externally managed, but require Tend to record desired + and active configuration digests and prove candidate/rollback compatibility. + +None of these options should permit arbitrary shell commands. Secret values +remain referenced server-side and must never enter release artifacts or state. + +## F-03: candidate fidelity + +### Observation + +A hardened singleton candidate receives the configured production environment +file and Tend's isolated listen-address override. It does not currently model +the installed systemd unit's complete argument vector. An application whose +configuration path is supplied by unit arguments may therefore start a +candidate with defaults instead of the intended production configuration. +That preserves live-state isolation, but it may prove only executable startup. +A missing credential, incompatible configuration field, filesystem permission, +or data migration requirement can then surface at activation time. + +### Options to assess + +- Define a fixed application preflight command or protocol that validates the + production configuration, credentials, permissions, and data compatibility + without binding the live port or mutating live state. +- Add a strict candidate argument vector to schema 2. Validate each argument, + reject secret values and paths outside the application contract, and pass it + directly to systemd without a shell. +- Permit an allowlisted `check` argument vector, never a shell string, with + bounded time/output and an explicitly non-mutating application contract. +- Record which checks ran against the binary alone and which ran against the + actual production configuration so the evidence cannot overstate coverage. + +## F-04: artifact review and approval ergonomics + +### Observation + +The secure flow deliberately separates build, review, digest approval, +transfer, candidate validation, and activation. v0.2 now produces a consistent +archive, manifest, SBOM, and checksum set, then transfers one exact artifact +through its restricted receiver. In practice, operators still need a +predictable way to discover that CI artifact, inspect it, approve exactly one +digest, and retain the review evidence. Without a first-class review path, +correct manual steps are easy to reconstruct differently for each application. + +### Options to assess + +- Add `tend inspect` for offline, non-mutating verification and a concise human + and JSON summary of source, build, dependency, and archive identities. +- Add an approval record that binds the artifact digest, service, target, + approver, and expiry without containing a credential. +- Standardize one Gitea artifact layout and documented download-to-activation + workflow. Keep production credentials unavailable to verification jobs. + +Approval must remain explicit; better ergonomics must not turn a successful +build into an automatic production mutation. + +## F-05: first installation and adoption + +### Observation + +The maintenance workflow expects an installed service, valid configuration, +and a current release pointer. `check-server` validates the restricted receiver +policy and prepared service boundary, but it does not create them. Observatory +bootstrap therefore required operator-managed service account, directories, +credentials, unit, configuration, and an initial current release before Tend +could own later maintenance safely. + +### Options to assess + +- `tend install`: a deliberately broader, separately approved transaction with + a strict schema and complete rollback of every supported created object. +- `tend adopt`: validate an existing service and release, copy or identify its + immutable artifact, establish state, and refuse ambiguous ownership. +- Keep bootstrap out of Tend, but ship a versioned acceptance checklist and a + machine-readable `check-server` result that maintenance can require. + +Installation and adoption must not be inferred from a missing state file. + +## F-06: release and configuration identity + +### Observation + +Content-addressed releases and v0.2 state make desired, candidate, active, +previous, and last-attempt binary identities clear. A live service is still a +combination of its binary, application configuration, credential bindings, +unit, routing fragment, and sometimes data schema. Tend cannot yet fully +explain that combined identity or determine whether a retained binary is +compatible with the current external configuration. + +### Options to assess + +- Extend the existing desired, candidate, active, previous, and last-attempt + release state with non-secret configuration and unit digests. +- Require rollback compatibility declarations or read-only application + preflight before changing traffic. +- Expose the identities and last validation results through + `tend status --json` for deployment evidence and future Observatory + ingestion. + +Configuration records contain digests and approved metadata only—not secret +contents. + +## F-07: offline module-cache completeness + +### Observation + +An exact Observatory package attempt used `GOPROXY=off` and a previously used +module cache. Source archives for the application dependencies were present, +but Go still needed several module metadata records while Tend enumerated the +complete build graph. Go refused the lookup and Tend stopped before building +or writing an artifact. Re-enabling the checksum-verified public proxy supplied +the missing metadata; the resulting package was byte-identical to the trusted +CI candidate. + +This is safe failure, but `GOPROXY=off` alone is not evidence of a hermetic +build. A cache can be partially populated even when ordinary builds happen to +succeed. + +### Options to assess + +- Add a resolver stage that runs with the pinned toolchain, proxy, and checksum + database, emits the complete module inventory, and materializes a bounded, + read-only cache for the builder. +- Run Tend's package stage with networking disabled and require every module, + checksum, source archive, and metadata record to come from that reviewed + cache. +- Add a non-mutating cache-completeness check that reports missing module + identities before the expensive double build. +- Consider a strictly verified vendored-source mode where repository size and + update review are acceptable; do not silently change dependency modes. + +Do not treat `GONOSUMDB`, `GOPRIVATE`, or a proxy bypass as an offline-build +solution. They alter verification or routing policy rather than proving cache +completeness. + +## F-08: linked-worktree VCS stamping + +### Observation + +An exact clean, pushed Observatory main commit passed verification and public +snapshot isolation from a detached linked Git worktree. Tend accepted its +source and remote provenance, then Go 1.26.6 stopped both `tend package` and an +equivalent direct build at the required `-buildvcs=true` step with `error +obtaining VCS status: exit status 128`. Ordinary Git status and commit queries +from the same worktree succeeded. + +A fresh standalone SSH clone of the identical commit packaged successfully +twice. Both archives were byte-identical and carried the expected commit, Go +version, VCS settings, checksums, manifest, and SBOM. This isolates the failure +to the linked-worktree build shape rather than the application source or module +graph. + +The failure is safe: Tend did not create a partial artifact and must not switch +to `-buildvcs=false`, because the package gate independently verifies the +embedded VCS revision and clean state. Until the interaction is resolved, use +a clean standalone clone for release packaging. + +### Options to assess + +- Detect a `.git` indirection file during `tend check` and fail before the + expensive double build with a precise standalone-clone instruction. +- Reproduce the interaction in a package integration test against the minimum + supported Go release and determine whether it is a Go toolchain limitation + or an invocation/environment defect. +- If linked worktrees can be supported, require the resulting build record to + carry the exact expected `vcs.revision` and `vcs.modified=false`; do not + synthesize those settings or disable VCS stamping. +- Consider an explicit, signed source-bundle input as part of the broader + source-attestation design. It must remain at least as strong as current + pushed-commit proof. + +## F-09: singleton traffic continuity + +### Observation + +During the August 18 Observatory maintenance exercise, the transient candidate +passed health, readiness, and content checks. Tend then stopped that candidate, +changed the singleton pointer, and restarted the installed fixed-address unit. +Caddy still targeted the fixed address during that restart, so the Observatory +agent observed a small number of transient HTTP `502` responses. Its durable +spool retried successfully and no accepted telemetry was lost, but the routed +origin was not continuously available. + +The corrected activation contract treats the candidate as a traffic handoff, +not merely a preflight process. Tend validates and routes the imported Caddy +handler to the candidate before changing the release pointer. It restarts and +probes the fixed-address unit without public traffic, routes back only after +that unit passes, and keeps the candidate healthy throughout the bounded +activation window. Any failure restores the former pointer and exact handler +bytes. If restoration itself cannot be completed, Tend leaves the proven +candidate routed and running for explicit operator recovery instead of causing +a known outage. + +Regression tests cover successful handoff, restart failure, public-origin +failure during the activation window, pointer and handler restoration, Caddy +validation/reload boundaries, and candidate cleanup. Final production evidence +must still repeat deploy, rollback, and reactivation with the exact release +binary before this finding is treated as released. + +## F-10: archive modes under a hardened umask + +### Observation + +The first two August 18 maintenance candidates stopped before activation. The +release archives correctly recorded executable mode `0755`, but extraction by +root under umask `0077` left the installed application binary mode `0700`. +The unprivileged transient candidate could not execute it. Tend emitted failed +candidate evidence, retained the prior release and route, and did not expose +the failed binary to traffic. + +Extraction now reapplies the already validated archive mode after file content +is closed. A Linux regression test sets umask `0077`, extracts the release, and +requires the installed binary to remain executable by the service identity. +The successful maintenance campaign used that corrected Tend binary. + +## F-12: retained-candidate reconciliation + +### Observation + +An Observatory activation failure intentionally left its proven handoff +candidate routed and running for operator recovery. A later push transferred +and validated a new content-addressed artifact, then stopped before candidate +startup because systemd still had Tend's fixed candidate unit loaded: + +`Unit observatory-tend-candidate.service was already loaded or has a fragment file.` + +The artifact, application, and host were valid. The collision was Tend-owned +residue from its own safe fallback behavior. Recovery required starting the +same stateless fallback under an independently named hardened unit, proving it, +routing Caddy to it, and only then stopping the old candidate and confirming +its port and unit name were free. No accepted application state was lost, but +the operator had to understand Tend's internal unit convention. + +### Implemented foundation + +- Transient candidate units are named by service plus the first 12 hexadecimal + characters of a fresh 128-bit operation identity. +- Conventional deployment state remains schema 1 so the retained v0.1 binary + can still read active and previous release identities. Singleton operations + write a separate strict, adjacent candidate-lease file containing the + operation, release, unit, address, and start time. +- `tend reconcile --json` reports persisted state, installed/candidate unit + activity, release pointers, and whether the exact handler file matches the + installed or candidate upstream. It explicitly distinguishes handler file + bytes from Caddy's loaded runtime configuration and performs no mutation. +- A retained forward-activation or rollback candidate lease survives an + incomplete recovery instead of being erased by a generic failed-attempt path. + Older binaries ignore the additive lease file, so operators must reconcile + before downgrading or attempting another activation. + +### Remaining options to assess + +- Extend the candidate lease with separately observed health and loaded-route + evidence rather than inferring either from a successful file write. +- Add an explicit `tend resume` operation that distinguishes active, routed, + rollback, abandoned, and unknown candidates without mutating by default. +- Permit automatic cleanup only after Tend proves that Caddy, current/previous + pointers, and the installed unit do not reference the candidate and that a + healthy route remains. +- Make a repeated activation of the same approved digest idempotently resume + the recorded operation instead of restaging or colliding with itself. + +Never resolve the collision by blindly stopping the loaded unit. A retained +candidate may be the only healthy route after a failed singleton activation. + +## F-13: public-origin topology and release identity + +### Observation + +During the same recovery, the public DNS record had already moved to a new +edge while the old host's Tend policy still named the canonical HTTPS origin +as its post-activation smoke target. A request from the old host would therefore +test a different deployment. It could fail even when the old host was healthy, +or pass against a matching marker served by the new host. Neither result proves +the release that Tend just activated. + +### Options to assess + +- Separate a local routed-origin probe—Caddy with the canonical Host/SNI on the + deployment host—from an external DNS-origin observation. +- Bind local success to a non-secret application release identity such as the + approved artifact digest, commit, and version, not only a human page marker. +- Record the expected public edge identity or resolved address set at approval + time. If it changes, classify the result as topology drift and require an + explicit migration decision instead of reporting an application failure. +- Allow independent external observers as additional evidence, but never let + a response from an unidentified deployment authorize activation. + +Public reachability remains valuable; it must be evidence about the intended +deployment rather than merely evidence that the hostname answered. + +## F-14: application data-path and preflight fidelity + +### Observation + +An exact, checksummed Observatory data copy was mounted at a different absolute +path in a container. Raw objects and durable SQLite databases were byte +identical, but the application catalogue intentionally compared stored segment +paths with filesystem-derived identities and failed closed. Preserving the +original in-container data path made the same Preview 15 binary ready in four +seconds with zero restarts and bounded memory. + +This was an application portability constraint, not a reason for Tend to +rewrite database state. It also demonstrates the limit of a binary-only +stateless candidate: executable health cannot prove compatibility with the +real configuration, credentials, mounts, or durable data. + +SQLite `-wal`, `-shm`, and process-lock files also changed during ordinary +open/close behavior while raw segments and durable databases did not. Migration +evidence should distinguish durable application truth from ephemeral runtime +coordination files. + +### Options to assess + +- Support a fixed, bounded application-owned preflight argument vector with an + explicit non-mutating contract, timeout, output limit, and no shell. +- Record the exact non-secret configuration digest, data-root identity, mount + identity, and checks performed alongside binary candidate evidence. +- Let applications define their durable migration evidence set; Tend should + transport and report those digests but must not infer database semantics or + edit state. +- Require rollback compatibility to be assessed against the same configuration + and data identities before an older binary is called safe. + +## F-15: facilitator rather than owner + +Tend's strict boundaries are valuable where authority changes hands: source +provenance, artifact digest approval, restricted transport, secret references, +path validation, traffic movement, and rollback. Strictness becomes harmful +when Tend's internal names or stale metadata are treated as application +requirements that an authorized maintainer must reverse-engineer. + +The target model is less invasive: + +- systemd units, Caddy handlers, environment files, release directories, and + application checks remain conventional and independently operable; +- Tend records an append-only migration/deployment journal and derives an + observed state before proposing mutation; +- schema migrations preserve old records and explain compatibility rather than + silently rejecting safe, recognizable state; +- `check`, `status`, and `reconcile` show facts and proposed repairs without + mutation; approval is required for artifact selection and traffic movement, + not for Tend implementation trivia; +- installation and adoption are explicit, while uninstalling Tend leaves an + understandable, runnable service and complete evidence trail. + +This does not relax hostile-input defenses. It moves strictness to the trust +boundary and makes recovery humane for the authenticated operator. + +## F-16: ambiguous container-network identity + +### Observation + +During the Observatory Preview 16 activation, the retained stateless handoff +container and the live Compose service both answered to the Docker network +alias `observatory`. A Caddy target such as `observatory:8093` could therefore +resolve to either release. Removing or restarting one container could also +change which release received traffic. The application and both containers +were healthy; the ambiguity existed solely in deployment identity. + +The activation used a separately validated Caddy fragment pinned to the exact +candidate address, proved its release marker before traffic moved, replaced +the live service, and then restored the reviewed application handler. No +ambiguous alias was used for the handoff. + +### Options to assess + +- Give every candidate an operation-scoped container name, network alias, and + lease that bind directly to its artifact digest and expected address. +- Resolve and inspect a proposed upstream immediately before Caddy validation; + reject zero, multiple, or identity-mismatched targets. +- Prefer a dedicated candidate network or an address supplied by the container + runtime over a stable alias shared with the live service. +- Record the exact routed target and release proof in activation state, then + verify that the restored production handler identifies the active release. + +Do not treat a healthy response from an ambiguous service name as release +proof. Availability and deployment identity are separate properties. + +## F-17: CI artifact publication is not artifact availability + +### Observation + +The trusted Observatory release-candidate workflow completed its two builds, +byte comparison, checksums, manifest, and SBOM, and its upload action reported +success. The forge artifact API subsequently returned no artifact for that +run. The deployment client therefore had no independently discoverable object +to download and inspect. Guessing a web-interface route or treating a green +upload step as possession of the bytes would have weakened the build-approve- +deploy boundary. + +The release was instead reproduced twice from a fresh, clean clone of the +exact pushed commit with the pinned toolchain. The packages compared +byte-for-byte, their manifest, checksums, SBOM, version, commit, and tree were +verified, and the approved digest was recorded. This preserved release +identity, but it is an operator workaround rather than the desired CI handoff. + +### Options to assess + +- Require the publisher to return a versioned receipt containing forge, run, + artifact identifier, size, digest, retention, and retrieval endpoint. +- Add a separate read-only verification step that downloads the artifact using + the same interface available to the approval client and reruns `tend + inspect` before the workflow is considered publish-complete. +- Support a content-addressed, append-only artifact store whose object name is + the approved SHA-256 and whose credentials remain separate from production. +- Let `tend push` accept only a locally present artifact plus an optional + verified publication receipt; never allow a successful CI status alone to + select bytes for deployment. + +Artifact availability must be proven from the consumer side. A successful +upload log is evidence of an attempted publication, not evidence that the +approved bytes can be recovered. + +The same failure repeated for Tend release-candidate run 369: the pinned job +successfully built identical archives, recorded digest +`2c225e0b9dd0be2d36fda62ab8d0b52cd9b28f9336c60dfb5edf3e715f450f95`, +and finalized a 3.18 MB upload, while both repository-wide and run-scoped REST +artifact listings returned no objects. An independent clean build reproduced +the exact digest before dogfooding. This confirms the problem is a reusable +forge-to-approval gap rather than an Observatory-specific packaging defect. + +## F-18: Docker Compose is not a systemd singleton + +### Observation + +Observatory Preview 18 was packaged reproducibly and its exact scratch image +passed a constrained loopback candidate check. The production service on +cliff-mads is a Docker Compose singleton, while Tend's supported strategies are +systemd/Caddy blue-green and systemd singleton-candidate. Tend was therefore +not used to pretend that an unsupported runtime had received a complete Tend +activation proof. + +The manual Compose replacement retained the former image and online database +backups and completed health, readiness, route, projection, agent, and log +checks. An independent public observer nevertheless recorded `200`, then +`502`/`503`, then `200` across an approximately 31.6-second replacement and +startup window. No application restart loop or accepted-data loss was +observed, but this is not continuous delivery. + +Preview.19 strengthened this finding. The exact candidate passed direct health +and readiness, but an activation script treated the first transient routed +`503` as final and attempted rollback before the proxy path had settled. The +previous binary then correctly rejected the newly migrated control schema and +entered a fail-closed restart loop. The operator recovered with the already +proven forward candidate while the agent remained paused. This was a bounded, +observable failure with intact backups, but it demonstrates that proxy +settling and data-schema rollback are different gates and cannot share one +generic failure branch. + +### Options to assess + +- Add a distinct, versioned Docker Compose strategy rather than arbitrary + command hooks or hidden container behavior inside existing strategies. +- Allocate an operation-scoped candidate container and network identity that + cannot collide with the production alias. +- Validate the exact image digest, non-secret Compose/configuration digest, + mounts, durable data identity, and application-owned read-only preflight + before moving traffic. +- Route Caddy to the proven candidate, replace or promote the production + service while the candidate remains healthy, then restore the reviewed live + handler only after the final container proves its release identity. +- Restore the prior handler and retained image automatically when any + activation-window probe fails; keep data rollback an explicit + application-owned decision. + +This strategy must remain optional. Tend should facilitate a conventional +Compose service without requiring Tend-only container labels, aliases, or +state for ordinary operator recovery. + +## F-19: binary rollback is not data rollback + +### Observation + +Observatory Preview.19 migrated its live control database from the schema +understood by Preview.18 to schema 11 before activation. When a premature +public-origin failure triggered binary rollback, Preview.18 rejected the new +schema rather than interpreting unknown state. That fail-closed behavior was +correct, but Tend-like orchestration cannot infer from two binary identities +whether their durable schemas are mutually readable. + +The exact pre-migration control and projection databases had already been +copied, integrity-checked, mode-restricted, and SHA-256 verified. The new +candidate had also passed the migration on an isolated copy. Because the +migration was forward-valid and the old agent was paused, completing the +Preview.19 activation preserved more verified state than restoring the backup. + +### Options to assess + +- Let an application declare a versioned, bounded compatibility statement for + current-to-candidate and candidate-to-previous data access. +- Treat migration completion as a journalled phase after which automatic + binary rollback is permitted only when backward readability was explicitly + proven. +- Support an application-owned, separately approved data-restoration plan; + never infer one from release pointers or run an arbitrary rollback hook. +- Keep verified backups and the proven forward candidate until the migration + soak closes, even when the previous binary remains retained. + +## F-20: stateful preflight needs its real resource envelope + +### Observation + +The Preview.19 stateless candidate and ordinary tests passed, but building a +presence-only SQLite index over a copied production projection failed with +`database or disk is full` under the service's 64 MiB `/tmp` tmpfs. The exact +same candidate and data passed with a 512 MiB tmpfs and drained all pending raw +segments. The projection database was healthy; the one-time index sort needed +more bounded scratch space than steady-state operation. + +### Options to assess + +- Record the non-secret service configuration digest and resource envelope + beside the artifact digest. +- Permit a strict application-owned preflight command selected from a reviewed + schema, never an arbitrary shell string. +- Exercise migrations against a copied or snapshot-backed production data set + under the candidate's exact CPU, memory, temporary-storage, filesystem, and + credential boundaries. +- Report resource exhaustion distinctly from corrupt data, failed health, or + incompatible schema so recovery guidance remains accurate. + +## Prioritization + +The recommended implementation order is: + +1. operation-scoped candidate leases plus explicit resume/repair built on the + new read-only reconciliation report; +2. migration compatibility and stateful-resource preflight contracts; +3. a bounded Docker Compose strategy with operation-scoped container/network + identities and unambiguous upstream + resolution and release proof; +4. release-bound local routed-origin proof separated from external DNS proof; +5. a bounded application preflight and configuration/data identity record; +6. a consumer-verifiable, digest-bound CI artifact publication receipt; +7. final production proof of the singleton traffic handoff; +8. a checksum-verified resolver and network-disabled package contract; +9. a provenance-preserving linked-worktree support decision after the new + early diagnostic; +10. a standardized CI artifact/approval contract building on `tend inspect`; +11. exact release plus configuration identity in status and state; +12. restricted transfer and receive with host policy; +13. explicit adoption for existing services; +14. managed configuration only after its restoration and failure-injection + model is as strong as binary activation. + +Friction entries should be updated with the implementing version, tests, and +dogfood evidence when resolved. Resolved entries remain in this ledger so the +reason for the security boundary is not lost. diff --git a/docs/PUBLIC_SNAPSHOT.md b/docs/PUBLIC_SNAPSHOT.md new file mode 100644 index 0000000..a13e257 --- /dev/null +++ b/docs/PUBLIC_SNAPSHOT.md @@ -0,0 +1,19 @@ +# Public snapshot boundary + +Private development happens in `gamertan/tend-dev`. The canonical public +repository is not a mirror of that Git history. `scripts/export-public.sh` +creates an exact-file, exact-commit, allowlisted filesystem snapshot and records +its source commit and tree in `PUBLIC-SNAPSHOT.json`. + +The exporter refuses dirty or unpushed source, destinations inside the source +or Git metadata, workflow directories, non-allowlisted paths, and known private +material markers. The resulting directory receives a new public root commit. + +Public Gitea is canonical for issues, contributions, and releases. GitHub is a +read-only discovery copy of the same public tree. Tags belong only to canonical +Gitea. + +The snapshot includes the program, security and architecture documentation, +copyable examples, release configuration, and local verification scripts. +Private workflows, runner configuration, repository credentials, and raw +operational evidence remain outside the public root. diff --git a/docs/SCHEMA_V2_MIGRATION.md b/docs/SCHEMA_V2_MIGRATION.md new file mode 100644 index 0000000..26ca6d4 --- /dev/null +++ b/docs/SCHEMA_V2_MIGRATION.md @@ -0,0 +1,44 @@ +# Schema 1 to schema 2 + +Configuration schema 2 and deployment state are separate versioned contracts. +The service configuration moves to schema 2, while conventional deployment +state deliberately remains schema 1 so the retained v0.1 binary can still read +active and previous release identities during recovery. + +Singleton operations write an adjacent, strict +`.candidate-lease.json` file containing the operation, release, +unit, address, and start time. Older binaries ignore that additive file. Do not +downgrade or start another deployment while a lease is present: first use the +new binary's `tend reconcile --json` report to establish which process and +route are healthy. Tend does not silently invent a lease for a legacy +interrupted operation. + +1. Move each configuration to `/etc/tend/services/.json`. +2. Set `schema_version` to `2`. +3. Add `service.environment_file` below `/etc/tend/environment/`. +4. Create that file as a regular root-owned file with mode `0600`. Move secret + values out of JSON. Do not put the environment file in Git. For a singleton, + keep its `listen_env` key out of this shared file: set the live address in + the installed unit and let Tend supply only the candidate address. +5. Set every service's `deployment.lock_file` to + `/run/lock/tend-deploy.lock`. +6. Add one or more query-free HTTPS `deployment.public_smoke` checks. +7. Add a per-service `deployment.event_log` below its release root and a + bounded `deployment.activation_window_seconds` value. Keep the log + root-owned and grant collectors read access explicitly. +8. For singleton services, add the full Caddy configuration, imported handler, + and one-upstream handler-template paths. The template must be reviewed and + contain exactly one `{{UPSTREAM}}` marker so Tend can keep traffic on the + candidate while the fixed-address unit restarts. +9. Update installed systemd units to read the same environment file as the + transient candidate. +10. Install a root-owned `0600` receive policy mapping each service name to its + exact configuration and artifact-size ceiling. +11. Run `tend check-server` as root before accepting a transfer. +12. Validate, activate, rollback, and reactivate one service at a time. Confirm + unrelated services never restart. + +Tend does not discover `.env`, infer old values, rewrite a production file, or +silently migrate state. For local development, copy a committed `.env.example` +to an ignored `.env.local`, restrict its mode, and load it with the application's +own tooling. Tend never loads local dotenv files implicitly. diff --git a/docs/THREAT_MODEL.md b/docs/THREAT_MODEL.md new file mode 100644 index 0000000..f9247c2 --- /dev/null +++ b/docs/THREAT_MODEL.md @@ -0,0 +1,55 @@ +# Threat model + +## Protected properties + +- A release is identified by an operator-approved SHA-256 digest. +- Only regular, bounded, allowlisted archive entries are extracted. +- Release roots, state, pointers, and Caddy files reject symlink substitution + at their checked boundaries. +- Configuration is strict JSON and is never interpolated into a shell command. +- Candidate health is established before traffic or any current release + pointer changes. New deployments also satisfy configured content smoke + checks; rollback uses health and readiness because future-release content + markers are not valid requirements for an older retained release. +- Caddy configuration validates before reload. +- Canonical routed origins and the previous blue/green slot or singleton + handoff candidate remain under probe for the configured activation window; a + failure restores the old handler and release pointer. +- Desired, candidate, active, previous, and failed-attempt identities remain + distinct in state. Bounded deployment events contain no arbitrary command + output or environment values and cannot block deployment. +- An activation failure restores the previously observed state. +- Active and previous releases survive pruning. +- The restricted receiver accepts one versioned bounded stream, one allowlisted + service name, and one explicitly approved digest. It accepts no remote path, + URL, environment value, or shell expression. +- OpenSSH host keys are pinned; forwarding, PTYs, proxy commands, local commands, + and user SSH configuration are disabled by the client invocation. +- Production secret values are absent from configuration, process arguments, + artifacts, manifests, reports, deployment state, and Tend logs. +- One host-wide lock serializes shared Caddy activation without stopping or + reconfiguring unrelated services. + +## Trusted inputs + +Reviewed source, the pinned Go toolchain, root-owned server policy and service +configuration, systemd unit files, the Caddy handler template, the operator, +and the target host's root account are trusted. Packaging a hostile repository +can execute hostile Go compiler hooks or consume resources; Tend is not a +source-code sandbox. + +## Adversarial inputs + +Artifact paths and bytes, protocol frames, requested service names, archive +metadata, stale or malformed state, HTTP responses, subprocess failures, and +filesystem objects at managed paths are validated and fail closed. Process +output and receive fields are bounded. + +## Non-goals and preview limits + +Tend does not defend against a concurrently malicious root user, a compromised +kernel/toolchain/systemd/Caddy/OpenSSH installation, denial of service by the +trusted application, or secrets an application itself exposes. It does not +manage data migrations, databases, containers, Kubernetes, or application-specific +activation. EQL therefore remains outside the generic deployment adapter until +its SQLite/catalog publication checkpoints can be modeled explicitly. diff --git a/docs/WALKTHROUGH.md b/docs/WALKTHROUGH.md new file mode 100644 index 0000000..4e8c0c2 --- /dev/null +++ b/docs/WALKTHROUGH.md @@ -0,0 +1,98 @@ +# Two-service build, approval, and deployment + +This walkthrough assumes one small Linux host running systemd and Caddy, two Go +services, and a trusted Gitea build runner. The host has the same root-owned Tend +binary at `/usr/local/bin/tend`; it does not need Git or Go. + +## Prepare the host once + +1. Create `/etc/tend/services`, `/etc/tend/environment`, and + `/var/lib/tend/incoming`. The incoming and environment directories are + root-owned mode `0700`. +2. Install one schema-2 file per service, one per-service deployment-event log, + and one root-owned mode-`0600` + environment file per service. A singleton's shared environment file must + not define its configured listen key; its installed unit owns the live + address and Tend overrides only the transient candidate. Give each + singleton an imported Caddy handler and a root-owned handler template with + exactly one `{{UPSTREAM}}` marker; this is the bounded traffic handoff while + its fixed-address unit restarts. +3. Install the receive policy, forced `authorized_keys` entry, and exact sudoers + rule from `examples/server/` after replacing every placeholder. The sudoers + fragment preserves only `SSH_ORIGINAL_COMMAND`; the root receiver requires + its exact protocol value and refuses every other requested command. +4. Pin the server host key in a dedicated client file. Do not accept a new key + interactively during deployment. +5. Run `sudo tend check-server` and inspect the allowlisted service names. +6. Run `sudo tend reconcile --config /etc/tend/services/example-site.json + --json` before the first maintenance release. A settled report is expected; + any retained candidate or unknown handler must be understood before traffic + changes. The command is read-only. + +## Build and approve + +Trusted CI checks a clean pushed commit, uses a pinned Go toolchain, packages +twice, and requires byte-identical archives. It publishes the archive, +`RELEASE.json`, `BUILDINFO.json`, SPDX SBOM, and SHA-256 evidence. + +The maintainer reads the candidate report and copies the exact approved digest +into the deployment command. Tend refuses a digest that is merely inferred from +the local file or differs from the produced value. + +Inspect the exact downloaded bytes before transfer. This is a complete, +read-only artifact validation and does not stage a candidate: + +```text +tend inspect --config /review/example-site.json \ + --artifact /approved/example-site.tar.gz \ + --sha256 --approve-sha256 +``` + +On the host, compare Tend's journal with the conventional service state before +and after activation. Reconciliation reports facts and never stops a unit, +rewrites a pointer, or changes Caddy: + +```text +sudo tend reconcile --config /etc/tend/services/example-site.json +``` + +```text +tend push --target tend-deploy@server.example \ + --known-hosts /secure/tend_known_hosts \ + --identity /secure/tend_deploy_ed25519 \ + --service example-site \ + --artifact /approved/example-site.tar.gz \ + --sha256 --approve-sha256 --activate +``` + +Repeat independently with `--service docs-site`. Both builds and transfers can +run concurrently. The host-wide lock serializes only activation because both +services share Caddy. Tend does not stop the other application. + +## Failure and recovery exercises + +- Change a candidate marker: activation must fail, preserve the active release, + and record the failed attempt without calling the candidate active. +- Make a Caddy template invalid: validation must fail and restore prior bytes. +- Make the public marker fail after an initially successful request: the + activation window must catch the transient routed failure and restore the + former slot/pointer. +- For a singleton, verify repeated canonical-origin requests remain successful + while Tend routes to the candidate, restarts the fixed-address unit, and + returns traffic to it. Inject failure at both Caddy reloads and require the + prior handler and pointer to be restored. +- Stop the previous blue/green slot during the activation window: Tend must + restore the old Caddy handler instead of accepting reduced continuity. +- Run `tend rollback --activate` for one service and verify the other service's + units, pointers, and public origin did not change. +- Interrupt a transfer: no release becomes active and the incomplete incoming + file is removed when the receiver exits. +- Interrupt singleton recovery after the candidate is proven. `tend reconcile + --json` must name the operation-scoped candidate unit, report whether it is + active and whether the handler file targets it, and perform no stop, restart, + reload, pointer, or state mutation. + +After the soak, prune per service. Active and previous releases remain protected. +If activation fails, run `reconcile` before manual recovery so the retained +candidate, route, release pointers, and journal disagreement are preserved in +one bounded report. diff --git a/examples/LICENSE b/examples/LICENSE new file mode 100644 index 0000000..540369f --- /dev/null +++ b/examples/LICENSE @@ -0,0 +1,12 @@ +Zero-Clause BSD + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..a1d4815 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,41 @@ +# Reusable Tend examples + +This subtree is licensed 0BSD so an operator can copy and adapt it without +bringing the Tend program's AGPL license into an application configuration. + +The blue/green example expects one separately reviewed environment file that +the two installed slots and transient validation use consistently: + +```text +# /etc/tend/environment/example-site.env +APP_SECRET=replace-on-server +``` + +The blue/green systemd slot units then read the nonsecret listen address from +`/etc/tend/slots/example-site-blue.env` or `-green.env`; Tend overrides only +the isolated candidate address. Secret values never enter `tend.json`. + +The singleton example follows the same split: its shared root-only environment +file omits the configured listen key, the installed unit owns the live address, +and Tend supplies only the transient candidate address. This prevents a shared +environment file from overriding the isolated candidate port. Its imported +Caddy handler is also managed from one reviewed template. Tend temporarily +routes the canonical origin to the proven candidate while the fixed-address +unit restarts, then returns traffic to that unit only after it passes its local +checks. + +Production configuration belongs outside the source checkout, owned by root, +and not group- or world-writable. The Caddy handler template is an entire +imported handler fragment; the enclosing site, matchers, and routing precedence +remain operator-owned. + +`event_log` is a per-service, root-owned JSONL evidence stream below that +service's release root. Grant an Observatory agent read access explicitly; do +not make the release root broadly readable. `activation_window_seconds` keeps +canonical routed probes active after Caddy reload and keeps the previous +blue/green slot—or the singleton handoff candidate—under health/readiness +observation until the activation is recorded. + +`server/` demonstrates the schema-2 receive policy, forced OpenSSH command, +restricted sudo entry, two independent service configurations, and secret-file +placement. The values are placeholders, not an installation script. diff --git a/examples/blue-green/caddy-handler.template b/examples/blue-green/caddy-handler.template new file mode 100644 index 0000000..aa4fe52 --- /dev/null +++ b/examples/blue-green/caddy-handler.template @@ -0,0 +1,2 @@ +# Managed by Tend. The enclosing site and route matchers remain operator-owned. +reverse_proxy {{UPSTREAM}} diff --git a/examples/blue-green/example-site@.service b/examples/blue-green/example-site@.service new file mode 100644 index 0000000..5b5f4cf --- /dev/null +++ b/examples/blue-green/example-site@.service @@ -0,0 +1,18 @@ +[Unit] +Description=Example site (%i) +After=network.target + +[Service] +Type=simple +DynamicUser=yes +ExecStart=/opt/example-site/slots/%i/example-site +EnvironmentFile=/etc/tend/environment/example-site.env +EnvironmentFile=/etc/tend/slots/example-site-%i.env +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=strict +ProtectHome=yes +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/examples/blue-green/tend.json b/examples/blue-green/tend.json new file mode 100644 index 0000000..641a1ff --- /dev/null +++ b/examples/blue-green/tend.json @@ -0,0 +1,26 @@ +{ + "schema_version": 2, + "service": { "name": "example-site", "allowed_host": "example.test", "environment_file": "/etc/tend/environment/example-site.env" }, + "build": { "package": "./cmd/site", "binary": "example-site", "branch": "main" }, + "deployment": { + "strategy": "blue_green", + "root": "/opt/example-site", + "lock_file": "/run/lock/tend-deploy.lock", + "state_file": "/opt/example-site/tend-state.json", + "event_log": "/opt/example-site/deployment-events.jsonl", + "health_path": "/healthz", + "readiness_path": "/readyz", + "candidate_timeout_seconds": 30, + "activation_window_seconds": 10, + "smoke": [{ "path": "/", "contains": "Example site" }], + "public_smoke": [{ "url": "https://example.test/", "contains": "Example site" }], + "blue_green": { + "caddy_config": "/etc/caddy/Caddyfile", + "caddy_handler": "/etc/caddy/example-site-handler.caddy", + "caddy_handler_template": "/etc/example-site/caddy-handler.template", + "bootstrap_active": "blue", + "blue": { "unit": "example-site-blue.service", "address": "127.0.0.1:8090", "link": "/opt/example-site/slots/blue" }, + "green": { "unit": "example-site-green.service", "address": "127.0.0.1:8091", "link": "/opt/example-site/slots/green" } + } + } +} diff --git a/examples/local/.env.example b/examples/local/.env.example new file mode 100644 index 0000000..54d24b5 --- /dev/null +++ b/examples/local/.env.example @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: 0BSD +APP_MODE=development +APP_SECRET=replace-with-a-local-random-value diff --git a/examples/server/authorized_keys.example b/examples/server/authorized_keys.example new file mode 100644 index 0000000..f6fc8ce --- /dev/null +++ b/examples/server/authorized_keys.example @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: 0BSD +restrict,command="sudo -n /usr/local/bin/tend receive --policy /etc/tend/receive-policy.json" ssh-ed25519 REPLACE_WITH_DEPLOY_KEY tend-deploy diff --git a/examples/server/caddy/docs-site.template b/examples/server/caddy/docs-site.template new file mode 100644 index 0000000..aa4fe52 --- /dev/null +++ b/examples/server/caddy/docs-site.template @@ -0,0 +1,2 @@ +# Managed by Tend. The enclosing site and route matchers remain operator-owned. +reverse_proxy {{UPSTREAM}} diff --git a/examples/server/caddy/example-site.template b/examples/server/caddy/example-site.template new file mode 100644 index 0000000..aa4fe52 --- /dev/null +++ b/examples/server/caddy/example-site.template @@ -0,0 +1,2 @@ +# Managed by Tend. The enclosing site and route matchers remain operator-owned. +reverse_proxy {{UPSTREAM}} diff --git a/examples/server/environment/docs-site.env.example b/examples/server/environment/docs-site.env.example new file mode 100644 index 0000000..36714c6 --- /dev/null +++ b/examples/server/environment/docs-site.env.example @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: 0BSD +DOCS_LISTEN=127.0.0.1:8102 +APP_SECRET=replace-on-server diff --git a/examples/server/environment/example-site.env.example b/examples/server/environment/example-site.env.example new file mode 100644 index 0000000..131fc81 --- /dev/null +++ b/examples/server/environment/example-site.env.example @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: 0BSD +APP_SECRET=replace-on-server diff --git a/examples/server/example-singleton.service b/examples/server/example-singleton.service new file mode 100644 index 0000000..ab416ac --- /dev/null +++ b/examples/server/example-singleton.service @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: 0BSD +[Unit] +Description=Example singleton site +After=network.target + +[Service] +Type=simple +DynamicUser=yes +ExecStart=/opt/example-site/current/example-site +EnvironmentFile=/etc/tend/environment/example-site.env +Environment=EXAMPLE_LISTEN=127.0.0.1:8092 +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=strict +ProtectHome=yes +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/examples/server/receive-policy.json b/examples/server/receive-policy.json new file mode 100644 index 0000000..3340fae --- /dev/null +++ b/examples/server/receive-policy.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "config_root": "/etc/tend/services", + "incoming_root": "/var/lib/tend/incoming", + "shared_lock_file": "/run/lock/tend-deploy.lock", + "services": { + "docs-site": { + "config": "/etc/tend/services/docs-site.json", + "max_artifact_bytes": 134217728 + }, + "example-site": { + "config": "/etc/tend/services/example-site.json", + "max_artifact_bytes": 134217728 + } + } +} diff --git a/examples/server/services/docs-site.json b/examples/server/services/docs-site.json new file mode 100644 index 0000000..5e7dc68 --- /dev/null +++ b/examples/server/services/docs-site.json @@ -0,0 +1,29 @@ +{ + "schema_version": 2, + "service": { "name": "docs-site", "allowed_host": "docs.example.test", "environment_file": "/etc/tend/environment/docs-site.env" }, + "build": { "package": "./cmd/docs", "binary": "docs-site", "branch": "main" }, + "deployment": { + "strategy": "singleton_candidate", + "root": "/opt/docs-site", + "lock_file": "/run/lock/tend-deploy.lock", + "state_file": "/opt/docs-site/tend-state.json", + "event_log": "/opt/docs-site/deployment-events.jsonl", + "health_path": "/healthz", + "readiness_path": "/readyz", + "candidate_timeout_seconds": 30, + "activation_window_seconds": 10, + "smoke": [{ "path": "/", "contains": "Documentation" }], + "public_smoke": [{ "url": "https://docs.example.test/", "contains": "Documentation" }], + "singleton": { + "unit": "docs-site.service", + "address": "127.0.0.1:8102", + "candidate_address": "127.0.0.1:18102", + "listen_env": "DOCS_LISTEN", + "current_link": "/opt/docs-site/current", + "previous_link": "/opt/docs-site/previous", + "caddy_config": "/etc/caddy/Caddyfile", + "caddy_handler": "/etc/caddy/docs-site-handler.caddy", + "caddy_handler_template": "/etc/tend/caddy/docs-site.template" + } + } +} diff --git a/examples/server/services/example-site.json b/examples/server/services/example-site.json new file mode 100644 index 0000000..b2f4fcf --- /dev/null +++ b/examples/server/services/example-site.json @@ -0,0 +1,29 @@ +{ + "schema_version": 2, + "service": { "name": "example-site", "allowed_host": "example.test", "environment_file": "/etc/tend/environment/example-site.env" }, + "build": { "package": "./cmd/site", "binary": "example-site", "branch": "main" }, + "deployment": { + "strategy": "singleton_candidate", + "root": "/opt/example-site", + "lock_file": "/run/lock/tend-deploy.lock", + "state_file": "/opt/example-site/tend-state.json", + "event_log": "/opt/example-site/deployment-events.jsonl", + "health_path": "/healthz", + "readiness_path": "/readyz", + "candidate_timeout_seconds": 30, + "activation_window_seconds": 10, + "smoke": [{ "path": "/", "contains": "Example site" }], + "public_smoke": [{ "url": "https://example.test/", "contains": "Example site" }], + "singleton": { + "unit": "example-site.service", + "address": "127.0.0.1:8092", + "candidate_address": "127.0.0.1:18092", + "listen_env": "EXAMPLE_LISTEN", + "current_link": "/opt/example-site/current", + "previous_link": "/opt/example-site/previous", + "caddy_config": "/etc/caddy/Caddyfile", + "caddy_handler": "/etc/caddy/example-site-handler.caddy", + "caddy_handler_template": "/etc/tend/caddy/example-site.template" + } + } +} diff --git a/examples/server/slots/example-site-blue.env b/examples/server/slots/example-site-blue.env new file mode 100644 index 0000000..1da346a --- /dev/null +++ b/examples/server/slots/example-site-blue.env @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: 0BSD +EXAMPLE_LISTEN=127.0.0.1:8090 diff --git a/examples/server/slots/example-site-green.env b/examples/server/slots/example-site-green.env new file mode 100644 index 0000000..57a9605 --- /dev/null +++ b/examples/server/slots/example-site-green.env @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: 0BSD +EXAMPLE_LISTEN=127.0.0.1:8091 diff --git a/examples/server/tend-receive.sudoers b/examples/server/tend-receive.sudoers new file mode 100644 index 0000000..9bfd7fd --- /dev/null +++ b/examples/server/tend-receive.sudoers @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: 0BSD +Defaults:tend-deploy env_keep += "SSH_ORIGINAL_COMMAND" +tend-deploy ALL=(root) NOPASSWD: /usr/local/bin/tend receive --policy /etc/tend/receive-policy.json diff --git a/examples/singleton/caddy-handler.template b/examples/singleton/caddy-handler.template new file mode 100644 index 0000000..aa4fe52 --- /dev/null +++ b/examples/singleton/caddy-handler.template @@ -0,0 +1,2 @@ +# Managed by Tend. The enclosing site and route matchers remain operator-owned. +reverse_proxy {{UPSTREAM}} diff --git a/examples/singleton/tend.json b/examples/singleton/tend.json new file mode 100644 index 0000000..b2f4fcf --- /dev/null +++ b/examples/singleton/tend.json @@ -0,0 +1,29 @@ +{ + "schema_version": 2, + "service": { "name": "example-site", "allowed_host": "example.test", "environment_file": "/etc/tend/environment/example-site.env" }, + "build": { "package": "./cmd/site", "binary": "example-site", "branch": "main" }, + "deployment": { + "strategy": "singleton_candidate", + "root": "/opt/example-site", + "lock_file": "/run/lock/tend-deploy.lock", + "state_file": "/opt/example-site/tend-state.json", + "event_log": "/opt/example-site/deployment-events.jsonl", + "health_path": "/healthz", + "readiness_path": "/readyz", + "candidate_timeout_seconds": 30, + "activation_window_seconds": 10, + "smoke": [{ "path": "/", "contains": "Example site" }], + "public_smoke": [{ "url": "https://example.test/", "contains": "Example site" }], + "singleton": { + "unit": "example-site.service", + "address": "127.0.0.1:8092", + "candidate_address": "127.0.0.1:18092", + "listen_env": "EXAMPLE_LISTEN", + "current_link": "/opt/example-site/current", + "previous_link": "/opt/example-site/previous", + "caddy_config": "/etc/caddy/Caddyfile", + "caddy_handler": "/etc/caddy/example-site-handler.caddy", + "caddy_handler_template": "/etc/tend/caddy/example-site.template" + } + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..806ba27 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module gamertan.com/tend + +go 1.26 diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..bc0571b --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,348 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package config + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/netip" + "net/url" + "os" + "path/filepath" + "regexp" + "strings" +) + +const ( + SchemaVersion = 2 + SharedLockFile = "/run/lock/tend-deploy.lock" +) + +var ( + namePattern = regexp.MustCompile(`^[a-z][a-z0-9-]{1,62}$`) + binaryPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) + packagePattern = regexp.MustCompile(`^\./[A-Za-z0-9_./-]+$`) + symbolPattern = regexp.MustCompile(`^[A-Za-z0-9._/-]+$`) + unitPattern = regexp.MustCompile(`^[A-Za-z0-9_.@-]+\.service$`) +) + +type Config struct { + SchemaVersion int `json:"schema_version"` + Service Service `json:"service"` + Build Build `json:"build"` + Deployment Deployment `json:"deployment"` +} + +type Service struct { + Name string `json:"name"` + AllowedHost string `json:"allowed_host"` + EnvironmentFile string `json:"environment_file"` +} + +type Build struct { + Package string `json:"package"` + Binary string `json:"binary"` + Branch string `json:"branch"` + VersionSymbol string `json:"version_symbol,omitempty"` + CommitSymbol string `json:"commit_symbol,omitempty"` + DateSymbol string `json:"date_symbol,omitempty"` +} + +type Deployment struct { + Strategy string `json:"strategy"` + Root string `json:"root"` + LockFile string `json:"lock_file"` + StateFile string `json:"state_file"` + EventLog string `json:"event_log"` + HealthPath string `json:"health_path"` + ReadinessPath string `json:"readiness_path"` + CandidateTimeoutSecs int `json:"candidate_timeout_seconds"` + ActivationWindowSecs int `json:"activation_window_seconds"` + Smoke []Smoke `json:"smoke"` + PublicSmoke []PublicSmoke `json:"public_smoke"` + BlueGreen *BlueGreen `json:"blue_green,omitempty"` + Singleton *Singleton `json:"singleton,omitempty"` +} + +type Smoke struct { + Path string `json:"path"` + Contains string `json:"contains"` +} + +type PublicSmoke struct { + URL string `json:"url"` + Contains string `json:"contains"` +} + +type BlueGreen struct { + CaddyConfig string `json:"caddy_config"` + CaddyHandler string `json:"caddy_handler"` + CaddyHandlerTemplate string `json:"caddy_handler_template"` + BootstrapActive string `json:"bootstrap_active"` + Blue Slot `json:"blue"` + Green Slot `json:"green"` +} + +type Slot struct { + Unit string `json:"unit"` + Address string `json:"address"` + Link string `json:"link"` +} + +type Singleton struct { + Unit string `json:"unit"` + Address string `json:"address"` + CandidateAddress string `json:"candidate_address"` + ListenEnv string `json:"listen_env"` + CurrentLink string `json:"current_link"` + PreviousLink string `json:"previous_link"` + CaddyConfig string `json:"caddy_config"` + CaddyHandler string `json:"caddy_handler"` + CaddyHandlerTemplate string `json:"caddy_handler_template"` +} + +func Load(path string) (Config, error) { + if !filepath.IsAbs(path) { + return Config{}, errors.New("configuration path must be absolute") + } + info, err := os.Lstat(path) + if err != nil { + return Config{}, fmt.Errorf("inspect configuration: %w", err) + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Size() > 1<<20 { + return Config{}, errors.New("configuration must be a bounded regular file, not a symlink") + } + b, err := os.ReadFile(path) + if err != nil { + return Config{}, fmt.Errorf("read configuration: %w", err) + } + dec := json.NewDecoder(bytes.NewReader(b)) + dec.DisallowUnknownFields() + var cfg Config + if err := dec.Decode(&cfg); err != nil { + return Config{}, fmt.Errorf("decode configuration: %w", err) + } + if err := requireEOF(dec); err != nil { + return Config{}, err + } + if err := cfg.Validate(); err != nil { + return Config{}, err + } + return cfg, nil +} + +func requireEOF(dec *json.Decoder) error { + var extra any + if err := dec.Decode(&extra); !errors.Is(err, io.EOF) { + if err == nil { + return errors.New("configuration contains multiple JSON values") + } + return fmt.Errorf("decode trailing configuration: %w", err) + } + return nil +} + +func (c Config) Validate() error { + if c.SchemaVersion != SchemaVersion { + return fmt.Errorf("schema_version must be %d", SchemaVersion) + } + if !namePattern.MatchString(c.Service.Name) { + return errors.New("service.name is invalid") + } + if c.Service.AllowedHost == "" || strings.ContainsAny(c.Service.AllowedHost, "/\\\x00\r\n\t ") { + return errors.New("service.allowed_host is invalid") + } + if err := safeAbsolute("service.environment_file", c.Service.EnvironmentFile); err != nil { + return err + } + if !within("/etc/tend/environment", c.Service.EnvironmentFile) { + return errors.New("service.environment_file must be below /etc/tend/environment") + } + if !packagePattern.MatchString(c.Build.Package) || strings.Contains(c.Build.Package, "..") { + return errors.New("build.package must be a local package without traversal") + } + if !binaryPattern.MatchString(c.Build.Binary) { + return errors.New("build.binary is invalid") + } + if c.Build.Branch == "" || strings.ContainsAny(c.Build.Branch, "\x00\r\n\t ~^:?*[\\") { + return errors.New("build.branch is invalid") + } + for label, symbol := range map[string]string{ + "build.version_symbol": c.Build.VersionSymbol, + "build.commit_symbol": c.Build.CommitSymbol, + "build.date_symbol": c.Build.DateSymbol, + } { + if symbol != "" && !symbolPattern.MatchString(symbol) { + return fmt.Errorf("%s is invalid", label) + } + } + d := c.Deployment + if err := safeAbsolute("deployment.root", d.Root); err != nil { + return err + } + if err := safeAbsolute("deployment.lock_file", d.LockFile); err != nil { + return err + } + if err := safeAbsolute("deployment.state_file", d.StateFile); err != nil { + return err + } + if filepath.Clean(d.StateFile) == filepath.Clean(d.Root) || !within(d.Root, d.StateFile) { + return errors.New("deployment.state_file must be below deployment.root") + } + if err := safeAbsolute("deployment.event_log", d.EventLog); err != nil { + return err + } + if filepath.Clean(d.EventLog) == filepath.Clean(d.Root) || !within(d.Root, d.EventLog) || filepath.Clean(d.EventLog) == filepath.Clean(d.StateFile) { + return errors.New("deployment.event_log must be a distinct file below deployment.root") + } + if !safeHTTPPath(d.HealthPath) || !safeHTTPPath(d.ReadinessPath) { + return errors.New("health and readiness paths must be absolute HTTP paths") + } + if d.CandidateTimeoutSecs < 2 || d.CandidateTimeoutSecs > 300 { + return errors.New("candidate_timeout_seconds must be between 2 and 300") + } + if d.ActivationWindowSecs < 1 || d.ActivationWindowSecs > 120 { + return errors.New("activation_window_seconds must be between 1 and 120") + } + if len(d.Smoke) == 0 || len(d.Smoke) > 32 { + return errors.New("deployment.smoke must contain 1 to 32 checks") + } + for i, smoke := range d.Smoke { + if !safeHTTPPath(smoke.Path) || smoke.Contains == "" || len(smoke.Contains) > 4096 || strings.ContainsRune(smoke.Contains, '\x00') { + return fmt.Errorf("deployment.smoke[%d] is invalid", i) + } + } + if len(d.PublicSmoke) == 0 || len(d.PublicSmoke) > 16 { + return errors.New("deployment.public_smoke must contain 1 to 16 checks") + } + for i, smoke := range d.PublicSmoke { + parsed, err := url.Parse(smoke.URL) + if err != nil || parsed.Scheme != "https" || parsed.Host == "" || parsed.User != nil || parsed.Fragment != "" || parsed.RawQuery != "" || parsed.Opaque != "" || smoke.Contains == "" || len(smoke.Contains) > 4096 || strings.ContainsRune(smoke.Contains, '\x00') { + return fmt.Errorf("deployment.public_smoke[%d] is invalid", i) + } + } + switch d.Strategy { + case "blue_green": + if d.BlueGreen == nil || d.Singleton != nil { + return errors.New("blue_green strategy requires only blue_green settings") + } + if err := validateBlueGreen(d.Root, *d.BlueGreen); err != nil { + return err + } + case "singleton_candidate": + if d.Singleton == nil || d.BlueGreen != nil { + return errors.New("singleton_candidate strategy requires only singleton settings") + } + if err := validateSingleton(d.Root, *d.Singleton); err != nil { + return err + } + default: + return errors.New("deployment.strategy must be blue_green or singleton_candidate") + } + return nil +} + +func validateBlueGreen(root string, b BlueGreen) error { + if b.BootstrapActive != "blue" && b.BootstrapActive != "green" { + return errors.New("blue_green.bootstrap_active must be blue or green") + } + for label, path := range map[string]string{"caddy_config": b.CaddyConfig, "caddy_handler": b.CaddyHandler, "caddy_handler_template": b.CaddyHandlerTemplate} { + if err := safeAbsolute("deployment.blue_green."+label, path); err != nil { + return err + } + } + if filepath.Clean(b.CaddyHandler) == filepath.Clean(b.CaddyHandlerTemplate) { + return errors.New("Caddy handler and template must be different files") + } + if err := validateSlot(root, "blue", b.Blue); err != nil { + return err + } + if err := validateSlot(root, "green", b.Green); err != nil { + return err + } + if b.Blue.Address == b.Green.Address || b.Blue.Unit == b.Green.Unit || b.Blue.Link == b.Green.Link { + return errors.New("blue and green slots must be distinct") + } + return nil +} + +func validateSlot(root, name string, slot Slot) error { + if !unitPattern.MatchString(slot.Unit) { + return fmt.Errorf("%s unit is invalid", name) + } + if err := loopbackAddress(slot.Address); err != nil { + return fmt.Errorf("%s address: %w", name, err) + } + if err := safeAbsolute(name+" link", slot.Link); err != nil { + return err + } + if !within(root, slot.Link) { + return fmt.Errorf("%s link must be below deployment.root", name) + } + return nil +} + +func validateSingleton(root string, s Singleton) error { + if !unitPattern.MatchString(s.Unit) { + return errors.New("singleton unit is invalid") + } + if err := loopbackAddress(s.Address); err != nil { + return fmt.Errorf("singleton address: %w", err) + } + if err := loopbackAddress(s.CandidateAddress); err != nil { + return fmt.Errorf("candidate address: %w", err) + } + if s.Address == s.CandidateAddress { + return errors.New("singleton addresses must be distinct") + } + if !regexp.MustCompile(`^[A-Z][A-Z0-9_]{0,63}$`).MatchString(s.ListenEnv) { + return errors.New("listen_env is invalid") + } + for _, entry := range []struct{ name, path string }{{"current_link", s.CurrentLink}, {"previous_link", s.PreviousLink}} { + if err := safeAbsolute(entry.name, entry.path); err != nil { + return err + } + if !within(root, entry.path) { + return fmt.Errorf("%s must be below deployment.root", entry.name) + } + } + if s.CurrentLink == s.PreviousLink { + return errors.New("current and previous links must differ") + } + for label, path := range map[string]string{"caddy_config": s.CaddyConfig, "caddy_handler": s.CaddyHandler, "caddy_handler_template": s.CaddyHandlerTemplate} { + if err := safeAbsolute("deployment.singleton."+label, path); err != nil { + return err + } + } + if filepath.Clean(s.CaddyHandler) == filepath.Clean(s.CaddyHandlerTemplate) { + return errors.New("singleton Caddy handler and template must be different files") + } + return nil +} + +func safeAbsolute(label, path string) error { + if !filepath.IsAbs(path) || filepath.Clean(path) != path || path == string(filepath.Separator) || strings.ContainsRune(path, '\x00') { + return fmt.Errorf("%s must be a clean, non-root absolute path", label) + } + return nil +} + +func within(root, child string) bool { + rel, err := filepath.Rel(filepath.Clean(root), filepath.Clean(child)) + return err == nil && rel != "." && rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator)) +} + +func safeHTTPPath(path string) bool { + return strings.HasPrefix(path, "/") && !strings.HasPrefix(path, "//") && !strings.ContainsAny(path, "\x00\r\n?#") +} + +func loopbackAddress(value string) error { + addr, err := netip.ParseAddrPort(value) + if err != nil || !addr.Addr().IsLoopback() || addr.Port() == 0 { + return errors.New("must be a loopback IP and nonzero port") + } + return nil +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..4f1b965 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,90 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package config + +import ( + "encoding/json" + "testing" +) + +func validConfig() Config { + return Config{ + SchemaVersion: 2, + Service: Service{Name: "example-site", AllowedHost: "example.test", EnvironmentFile: "/etc/tend/environment/example-site.env"}, + Build: Build{Package: "./cmd/site", Binary: "example-site", Branch: "main"}, + Deployment: Deployment{ + Strategy: "blue_green", Root: "/opt/example-site", LockFile: SharedLockFile, + StateFile: "/opt/example-site/state.json", EventLog: "/opt/example-site/deployment-events.jsonl", HealthPath: "/healthz", ReadinessPath: "/readyz", + CandidateTimeoutSecs: 30, ActivationWindowSecs: 10, Smoke: []Smoke{{Path: "/", Contains: "Example"}}, + PublicSmoke: []PublicSmoke{{URL: "https://example.test/", Contains: "Example"}}, + BlueGreen: &BlueGreen{ + CaddyConfig: "/etc/caddy/Caddyfile", CaddyHandler: "/etc/caddy/example.caddy", + CaddyHandlerTemplate: "/etc/example/caddy.template", + BootstrapActive: "blue", + Blue: Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: "/opt/example-site/slots/blue"}, + Green: Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: "/opt/example-site/slots/green"}, + }, + }, + } +} + +func TestValidateAcceptsBlueGreen(t *testing.T) { + if err := validConfig().Validate(); err != nil { + t.Fatal(err) + } +} + +func TestValidateSingletonRequiresDistinctCaddyHandoffFiles(t *testing.T) { + cfg := validConfig() + cfg.Deployment.Strategy = "singleton_candidate" + cfg.Deployment.BlueGreen = nil + cfg.Deployment.Singleton = &Singleton{ + Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN", + CurrentLink: "/opt/example-site/current", PreviousLink: "/opt/example-site/previous", CaddyConfig: "/etc/caddy/Caddyfile", + CaddyHandler: "/etc/caddy/example-site.caddy", CaddyHandlerTemplate: "/etc/tend/caddy/example-site.template", + } + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + cfg.Deployment.Singleton.CaddyHandlerTemplate = cfg.Deployment.Singleton.CaddyHandler + if err := cfg.Validate(); err == nil { + t.Fatal("expected shared handler/template path to be rejected") + } +} + +func TestValidateRejectsHostileValues(t *testing.T) { + tests := map[string]func(*Config){ + "unknown strategy": func(c *Config) { c.Deployment.Strategy = "shell" }, + "nonloopback": func(c *Config) { c.Deployment.BlueGreen.Blue.Address = "203.0.113.7:80" }, + "root path": func(c *Config) { c.Deployment.Root = "/" }, + "traversal": func(c *Config) { c.Build.Package = "./cmd/../secret" }, + "shared slot": func(c *Config) { c.Deployment.BlueGreen.Green.Link = c.Deployment.BlueGreen.Blue.Link }, + "bad smoke": func(c *Config) { c.Deployment.Smoke[0].Path = "https://attacker.test/" }, + "bad public smoke": func(c *Config) { c.Deployment.PublicSmoke[0].URL = "http://example.test/" }, + "public secret query": func(c *Config) { c.Deployment.PublicSmoke[0].URL = "https://example.test/?token=secret" }, + "environment sibling": func(c *Config) { c.Service.EnvironmentFile = "/etc/tend/environment-old/example.env" }, + } + for name, mutate := range tests { + t.Run(name, func(t *testing.T) { + cfg := validConfig() + mutate(&cfg) + if err := cfg.Validate(); err == nil { + t.Fatal("expected validation failure") + } + }) + } +} + +func TestUnknownJSONFieldRejected(t *testing.T) { + b, err := json.Marshal(validConfig()) + if err != nil { + t.Fatal(err) + } + var raw map[string]any + if err := json.Unmarshal(b, &raw); err != nil { + t.Fatal(err) + } + raw["surprise"] = true + b, _ = json.Marshal(raw) + _ = b // Load exercises strict decoding from disk in command tests. +} diff --git a/internal/deploy/deploy.go b/internal/deploy/deploy.go new file mode 100644 index 0000000..6002046 --- /dev/null +++ b/internal/deploy/deploy.go @@ -0,0 +1,1023 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package deploy + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "gamertan.com/tend/internal/config" + "gamertan.com/tend/internal/eventlog" + "gamertan.com/tend/internal/state" +) + +type Request struct { + Artifact string + SHA256 string + ApprovedSHA256 string + Activate bool +} +type Report struct { + Validated bool `json:"validated"` + Mutation string `json:"mutation"` + Release string `json:"release,omitempty"` + ActiveRelease string `json:"active_release,omitempty"` + PreviousRelease string `json:"previous_release,omitempty"` + EventWarnings int `json:"event_warnings,omitempty"` + LeaseCleanupPending bool `json:"lease_cleanup_pending,omitempty"` +} +type Status struct { + State *state.Record `json:"state,omitempty"` + Units map[string]bool `json:"units"` + StateInitialized bool `json:"state_initialized"` +} + +type retainedCandidateError struct{ cause error } + +func (e *retainedCandidateError) Error() string { return e.cause.Error() } +func (e *retainedCandidateError) Unwrap() error { return e.cause } + +type Manager struct { + Operator Operator + Now func() time.Time + Prepare func(config.Config, string, string, string) (string, error) + Inspect func(config.Config, string, string, string) error + ReadIdentity func(string) (releaseIdentity, error) + OperationID func() (string, error) + AppendEvent func(string, eventlog.Event) error + Sleep func(context.Context, time.Duration) error +} + +func NewManager(operator Operator) Manager { + return Manager{Operator: operator, Now: time.Now, Prepare: prepareRelease, Inspect: inspectArtifact, ReadIdentity: readReleaseIdentity, OperationID: eventlog.OperationID, AppendEvent: eventlog.Append, Sleep: sleepContext} +} + +func (m Manager) Deploy(ctx context.Context, cfg config.Config, request Request) (Report, error) { + if err := cfg.Validate(); err != nil { + return Report{}, err + } + if !request.Activate { + if err := m.Inspect(cfg, request.Artifact, request.SHA256, request.ApprovedSHA256); err != nil { + return Report{}, err + } + return Report{Validated: true, Mutation: "none"}, nil + } + lock, err := acquireLock(cfg.Deployment.LockFile) + if err != nil { + return Report{}, err + } + defer lock.Close() + release, err := m.Prepare(cfg, request.Artifact, request.SHA256, request.ApprovedSHA256) + if err != nil { + return Report{}, err + } + started := m.Now() + eventWarnings := 0 + identity, identityErr := m.ReadIdentity(release) + if identityErr != nil { + eventWarnings++ + } + operationID := "" + if m.OperationID != nil { + operationID, err = m.OperationID() + if err != nil { + eventWarnings++ + operationID = "" + } + } + if cfg.Deployment.Strategy == "singleton_candidate" && operationID == "" { + return Report{}, errors.New("singleton activation requires a fresh operation identity") + } + emit := func(phase, slot, outcome string) { + if m.AppendEvent == nil || identityErr != nil || operationID == "" { + return + } + event := eventlog.Event{Version: eventlog.Version, OperationID: operationID, Service: cfg.Service.Name, ArtifactDigest: request.ApprovedSHA256, Commit: identity.Commit, ReleaseVersion: identity.Version, Phase: phase, Slot: slot, DurationMillis: max(0, m.Now().Sub(started).Milliseconds()), Outcome: outcome, ObservedAt: m.Now().UTC().Format(time.RFC3339Nano)} + if eventErr := m.AppendEvent(cfg.Deployment.EventLog, event); eventErr != nil { + eventWarnings++ + } + } + record, err := loadOrBootstrap(cfg, m.Now()) + if err != nil { + return Report{}, err + } + leasePath := state.CandidateLeasePath(cfg.Deployment.StateFile) + var candidateLease state.CandidateLease + if cfg.Deployment.Strategy == "singleton_candidate" { + _, exists, leaseErr := loadCandidateLease(cfg) + if leaseErr != nil { + return Report{}, leaseErr + } + if record.CandidateRelease != "" || exists { + return Report{}, errors.New("singleton candidate lease is unresolved; run tend reconcile --json before another activation") + } + } + attemptAt := m.Now().UTC().Format(time.RFC3339) + record.DesiredRelease = release + record.CandidateRelease = release + record.LastAttemptRelease = release + record.LastAttemptOutcome = "running" + record.LastAttemptAt = attemptAt + record.UpdatedAt = attemptAt + if cfg.Deployment.Strategy == "singleton_candidate" { + candidateUnit, unitErr := singletonCandidateUnit(cfg.Service.Name, operationID) + if unitErr != nil { + return Report{}, unitErr + } + candidateLease = state.CandidateLease{SchemaVersion: state.CandidateLeaseSchemaVersion, Service: cfg.Service.Name, OperationID: operationID, Release: release, Unit: candidateUnit, Address: cfg.Deployment.Singleton.CandidateAddress, StartedAt: attemptAt} + } + if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil { + return Report{}, err + } + if cfg.Deployment.Strategy == "singleton_candidate" { + if err := state.StoreCandidateLease(leasePath, cfg.Deployment.Root, candidateLease); err != nil { + failed := record + clearCandidateLease(&failed) + failed.LastAttemptOutcome = "failed" + failed.UpdatedAt = m.Now().UTC().Format(time.RFC3339) + if storeErr := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, failed); storeErr != nil { + return Report{}, errors.Join(err, storeErr) + } + return Report{}, err + } + } + emit("candidate", inactiveSlot(cfg, record), "running") + switch cfg.Deployment.Strategy { + case "blue_green": + err = m.deployBlueGreen(ctx, cfg, record, release) + case "singleton_candidate": + err = m.deploySingleton(ctx, cfg, record, release, candidateLease.Unit) + default: + err = errors.New("unsupported strategy") + } + if err != nil { + failed := record + var retained *retainedCandidateError + if !errors.As(err, &retained) { + if cleanupErr := state.RemoveCandidateLease(leasePath); cleanupErr != nil { + err = errors.Join(err, fmt.Errorf("candidate lease cleanup failed: %w", cleanupErr)) + } else { + clearCandidateLease(&failed) + } + } + failed.LastAttemptOutcome = "failed" + failed.UpdatedAt = m.Now().UTC().Format(time.RFC3339) + if storeErr := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, failed); storeErr != nil { + err = errors.Join(err, storeErr) + } + emit("activation", inactiveSlot(cfg, record), "failed") + return Report{EventWarnings: eventWarnings}, err + } + emit("activation", inactiveSlot(cfg, record), "succeeded") + updated, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if err != nil { + return Report{}, err + } + report := Report{Validated: true, Mutation: "activated", Release: release, ActiveRelease: updated.ActiveRelease, PreviousRelease: updated.PreviousRelease, EventWarnings: eventWarnings} + if cfg.Deployment.Strategy == "singleton_candidate" { + if cleanupErr := state.RemoveCandidateLease(leasePath); cleanupErr != nil { + report.LeaseCleanupPending = true + } + } + return report, nil +} + +func singletonCandidateUnit(service, operationID string) (string, error) { + if len(operationID) != 32 { + return "", errors.New("candidate operation identity is invalid") + } + for _, character := range operationID { + if !strings.ContainsRune("0123456789abcdef", character) { + return "", errors.New("candidate operation identity is invalid") + } + } + return service + "-tend-candidate-" + operationID[:12] + ".service", nil +} + +func clearCandidateLease(record *state.Record) { + record.CandidateRelease = "" +} + +func loadCandidateLease(cfg config.Config) (state.CandidateLease, bool, error) { + lease, err := state.LoadCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, cfg.Service.Name) + if err == nil { + return lease, true, nil + } + if os.IsNotExist(err) { + return state.CandidateLease{}, false, nil + } + return state.CandidateLease{}, false, fmt.Errorf("load singleton candidate lease: %w", err) +} + +type releaseIdentity struct { + Version string `json:"version"` + Commit string `json:"commit"` +} + +func readReleaseIdentity(release string) (releaseIdentity, error) { + b, err := os.ReadFile(filepath.Join(release, "RELEASE.json")) + if err != nil { + return releaseIdentity{}, fmt.Errorf("read installed release identity: %w", err) + } + if len(b) > 1<<20 { + return releaseIdentity{}, errors.New("installed release identity is too large") + } + var identity releaseIdentity + if err := json.Unmarshal(b, &identity); err != nil { + return releaseIdentity{}, errors.New("decode installed release identity") + } + if identity.Version == "" || identity.Commit == "" { + return releaseIdentity{}, errors.New("installed release identity is incomplete") + } + return identity, nil +} + +func inactiveSlot(cfg config.Config, record state.Record) string { + if cfg.Deployment.Strategy == "singleton_candidate" { + return "singleton" + } + if record.ActiveSlot == "blue" { + return "green" + } + return "blue" +} + +func sleepContext(ctx context.Context, duration time.Duration) error { + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func loadOrBootstrap(cfg config.Config, now time.Time) (state.Record, error) { + record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if err == nil { + return record, nil + } + if !os.IsNotExist(err) { + return state.Record{}, err + } + switch cfg.Deployment.Strategy { + case "blue_green": + slot := cfg.Deployment.BlueGreen.BootstrapActive + release, err := resolveReleaseLink(cfg.Deployment.Root, slotConfig(*cfg.Deployment.BlueGreen, slot).Link) + if err != nil { + return state.Record{}, fmt.Errorf("bootstrap active slot: %w", err) + } + return state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: slot, ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil + case "singleton_candidate": + release, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink) + if err != nil { + return state.Record{}, fmt.Errorf("bootstrap singleton: %w", err) + } + return state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, UpdatedAt: now.UTC().Format(time.RFC3339)}, nil + } + return state.Record{}, errors.New("unsupported strategy") +} + +func (m Manager) deployBlueGreen(ctx context.Context, cfg config.Config, record state.Record, release string) (err error) { + bg := *cfg.Deployment.BlueGreen + inactive := "blue" + if record.ActiveSlot == "blue" { + inactive = "green" + } + slot := slotConfig(bg, inactive) + oldInactive, oldErr := resolveReleaseLink(cfg.Deployment.Root, slot.Link) + if oldErr != nil && !os.IsNotExist(oldErr) { + return oldErr + } + oldHandler, err := os.ReadFile(bg.CaddyHandler) + if err != nil { + return fmt.Errorf("read current Caddy handler: %w", err) + } + handlerChanged := false + linkChanged := false + defer func() { + if err == nil { + return + } + if handlerChanged { + _ = atomicWrite(bg.CaddyHandler, oldHandler, 0o644) + _ = m.Operator.ValidateCaddy(ctx, bg.CaddyConfig) + _ = m.Operator.ReloadCaddy(ctx) + } + if linkChanged { + if oldErr == nil { + _ = replaceSymlink(slot.Link, oldInactive) + _ = m.Operator.Restart(ctx, slot.Unit) + } else { + _ = removeSymlink(slot.Link) + _ = m.Operator.Stop(ctx, slot.Unit) + } + } + }() + if err = replaceSymlink(slot.Link, release); err != nil { + return err + } + linkChanged = true + if err = m.Operator.Restart(ctx, slot.Unit); err != nil { + return err + } + if err = m.probeAll(ctx, cfg, slot.Address); err != nil { + return fmt.Errorf("candidate failed: %w", err) + } + handler, err := renderHandler(bg.CaddyHandlerTemplate, slot.Address) + if err != nil { + return err + } + if err = atomicWrite(bg.CaddyHandler, handler, 0o644); err != nil { + return err + } + handlerChanged = true + if err = m.Operator.ValidateCaddy(ctx, bg.CaddyConfig); err != nil { + return fmt.Errorf("Caddy validation failed: %w", err) + } + if err = m.Operator.ReloadCaddy(ctx); err != nil { + return fmt.Errorf("Caddy reload failed: %w", err) + } + if err = m.probeAll(ctx, cfg, slot.Address); err != nil { + return fmt.Errorf("post-activation smoke failed: %w", err) + } + if err = m.probePublic(ctx, cfg, true); err != nil { + return fmt.Errorf("public-origin smoke failed: %w", err) + } + previous := slotConfig(bg, record.ActiveSlot) + if err = m.continuityWindow(ctx, cfg, previous.Address, true); err != nil { + return fmt.Errorf("activation continuity failed: %w", err) + } + next := state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: inactive, ActiveRelease: release, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, LastAttemptRelease: release, LastAttemptOutcome: "succeeded", LastAttemptAt: record.LastAttemptAt, UpdatedAt: m.Now().UTC().Format(time.RFC3339)} + if err = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next); err != nil { + return err + } + return nil +} + +func (m Manager) deploySingleton(ctx context.Context, cfg config.Config, record state.Record, release, candidateUnit string) (err error) { + if err = m.activateSingletonRelease(ctx, cfg, release, candidateUnit, true); err != nil { + return err + } + next := state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, LastAttemptRelease: release, LastAttemptOutcome: "succeeded", LastAttemptAt: record.LastAttemptAt, UpdatedAt: m.Now().UTC().Format(time.RFC3339)} + if err = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next); err != nil { + return err + } + return nil +} + +func (m Manager) Rollback(ctx context.Context, cfg config.Config) (state.Record, error) { + lock, err := acquireLock(cfg.Deployment.LockFile) + if err != nil { + return state.Record{}, err + } + defer lock.Close() + record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if err != nil { + return state.Record{}, err + } + if record.PreviousRelease == "" { + return state.Record{}, errors.New("no previous release is recorded") + } + leasePath := state.CandidateLeasePath(cfg.Deployment.StateFile) + if cfg.Deployment.Strategy == "singleton_candidate" { + _, exists, leaseErr := loadCandidateLease(cfg) + if leaseErr != nil { + return state.Record{}, leaseErr + } + if record.CandidateRelease != "" || exists { + return state.Record{}, errors.New("singleton candidate lease is unresolved; run tend reconcile --json before rollback") + } + } + started := m.Now() + identity, identityErr := m.ReadIdentity(record.PreviousRelease) + digest, digestErr := releaseDigest(record.PreviousRelease) + operationID := "" + if m.OperationID != nil { + operationID, err = m.OperationID() + } + if cfg.Deployment.Strategy == "singleton_candidate" && (err != nil || operationID == "") { + return state.Record{}, errors.New("singleton rollback requires a fresh operation identity") + } + candidateUnit := "" + rollbackAttempt := record + if cfg.Deployment.Strategy == "singleton_candidate" { + candidateUnit, err = singletonCandidateUnit(cfg.Service.Name, operationID) + if err != nil { + return state.Record{}, err + } + attemptAt := m.Now().UTC().Format(time.RFC3339) + rollbackAttempt.DesiredRelease = record.PreviousRelease + rollbackAttempt.CandidateRelease = record.PreviousRelease + rollbackAttempt.LastAttemptRelease = record.PreviousRelease + rollbackAttempt.LastAttemptOutcome = "running" + rollbackAttempt.LastAttemptAt = attemptAt + rollbackAttempt.UpdatedAt = attemptAt + if err = state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, rollbackAttempt); err != nil { + return state.Record{}, err + } + lease := state.CandidateLease{SchemaVersion: state.CandidateLeaseSchemaVersion, Service: cfg.Service.Name, OperationID: operationID, Release: record.PreviousRelease, Unit: candidateUnit, Address: cfg.Deployment.Singleton.CandidateAddress, StartedAt: attemptAt} + if err = state.StoreCandidateLease(leasePath, cfg.Deployment.Root, lease); err != nil { + failed := rollbackAttempt + clearCandidateLease(&failed) + failed.LastAttemptOutcome = "failed" + failed.UpdatedAt = m.Now().UTC().Format(time.RFC3339) + if storeErr := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, failed); storeErr != nil { + return state.Record{}, errors.Join(err, storeErr) + } + return state.Record{}, err + } + } + emit := func(outcome string) { + if m.AppendEvent == nil || identityErr != nil || digestErr != nil || operationID == "" { + return + } + event := eventlog.Event{Version: eventlog.Version, OperationID: operationID, Service: cfg.Service.Name, ArtifactDigest: digest, Commit: identity.Commit, ReleaseVersion: identity.Version, Phase: "rollback", Slot: record.PreviousSlot, DurationMillis: max(0, m.Now().Sub(started).Milliseconds()), Outcome: outcome, ObservedAt: m.Now().UTC().Format(time.RFC3339Nano)} + _ = m.AppendEvent(cfg.Deployment.EventLog, event) + } + emit("running") + switch cfg.Deployment.Strategy { + case "blue_green": + err = m.rollbackBlueGreen(ctx, cfg, record) + case "singleton_candidate": + err = m.rollbackSingleton(ctx, cfg, record, candidateUnit) + default: + err = errors.New("unsupported strategy") + } + if err != nil { + if cfg.Deployment.Strategy == "singleton_candidate" { + failed := rollbackAttempt + var retained *retainedCandidateError + if !errors.As(err, &retained) { + if cleanupErr := state.RemoveCandidateLease(leasePath); cleanupErr != nil { + err = errors.Join(err, fmt.Errorf("candidate lease cleanup failed: %w", cleanupErr)) + } else { + clearCandidateLease(&failed) + } + } + failed.LastAttemptOutcome = "failed" + failed.UpdatedAt = m.Now().UTC().Format(time.RFC3339) + if storeErr := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, failed); storeErr != nil { + err = errors.Join(err, storeErr) + } + } + emit("failed") + return state.Record{}, err + } + emit("succeeded") + updated, loadErr := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if loadErr != nil { + return state.Record{}, loadErr + } + if cfg.Deployment.Strategy == "singleton_candidate" { + if cleanupErr := state.RemoveCandidateLease(leasePath); cleanupErr != nil { + return updated, fmt.Errorf("rollback succeeded but candidate lease cleanup is pending; run tend reconcile --json: %w", cleanupErr) + } + } + return updated, nil +} + +func releaseDigest(release string) (string, error) { + name := filepath.Base(release) + if !strings.HasPrefix(name, "sha256-") { + return "", errors.New("release is not content addressed") + } + digest := strings.TrimPrefix(name, "sha256-") + if len(digest) != 64 { + return "", errors.New("release digest is invalid") + } + for _, character := range digest { + if !strings.ContainsRune("0123456789abcdef", character) { + return "", errors.New("release digest is invalid") + } + } + return digest, nil +} +func (m Manager) rollbackBlueGreen(ctx context.Context, cfg config.Config, record state.Record) (err error) { + bg := *cfg.Deployment.BlueGreen + slot := slotConfig(bg, record.PreviousSlot) + if active, checkErr := m.Operator.IsActive(ctx, slot.Unit); checkErr != nil { + return checkErr + } else if !active { + if err = m.Operator.Restart(ctx, slot.Unit); err != nil { + return err + } + } + if err = m.probeHealthReadiness(ctx, cfg, slot.Address); err != nil { + return err + } + oldHandler, err := os.ReadFile(bg.CaddyHandler) + if err != nil { + return err + } + changed := false + defer func() { + if err != nil && changed { + _ = atomicWrite(bg.CaddyHandler, oldHandler, 0o644) + _ = m.Operator.ValidateCaddy(ctx, bg.CaddyConfig) + _ = m.Operator.ReloadCaddy(ctx) + } + }() + handler, err := renderHandler(bg.CaddyHandlerTemplate, slot.Address) + if err != nil { + return err + } + if err = atomicWrite(bg.CaddyHandler, handler, 0o644); err != nil { + return err + } + changed = true + if err = m.Operator.ValidateCaddy(ctx, bg.CaddyConfig); err != nil { + return err + } + if err = m.Operator.ReloadCaddy(ctx); err != nil { + return err + } + if err = m.probeHealthReadiness(ctx, cfg, slot.Address); err != nil { + return err + } + if err = m.probePublic(ctx, cfg, false); err != nil { + return err + } + next := state.Record{SchemaVersion: state.SchemaVersion, Strategy: record.Strategy, DesiredRelease: record.PreviousRelease, ActiveSlot: record.PreviousSlot, ActiveRelease: record.PreviousRelease, PreviousSlot: record.ActiveSlot, PreviousRelease: record.ActiveRelease, LastAttemptRelease: record.PreviousRelease, LastAttemptOutcome: "rolled_back", LastAttemptAt: m.Now().UTC().Format(time.RFC3339), UpdatedAt: m.Now().UTC().Format(time.RFC3339)} + return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next) +} +func (m Manager) rollbackSingleton(ctx context.Context, cfg config.Config, record state.Record, candidateUnit string) (err error) { + if err = m.activateSingletonRelease(ctx, cfg, record.PreviousRelease, candidateUnit, false); err != nil { + return err + } + next := state.Record{SchemaVersion: state.SchemaVersion, Strategy: record.Strategy, DesiredRelease: record.PreviousRelease, ActiveSlot: "singleton", ActiveRelease: record.PreviousRelease, PreviousSlot: "singleton", PreviousRelease: record.ActiveRelease, LastAttemptRelease: record.PreviousRelease, LastAttemptOutcome: "rolled_back", LastAttemptAt: m.Now().UTC().Format(time.RFC3339), UpdatedAt: m.Now().UTC().Format(time.RFC3339)} + return state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, next) +} + +// activateSingletonRelease keeps public traffic on a proven process while the +// installed fixed-address unit changes release. The transient candidate first +// receives traffic, remains healthy through the handoff, and is stopped only +// after Caddy points back to the verified installed unit. +func (m Manager) activateSingletonRelease(ctx context.Context, cfg config.Config, release, candidateUnit string, checkMarkers bool) (err error) { + single := *cfg.Deployment.Singleton + env := map[string]string{single.ListenEnv: single.CandidateAddress} + binary := filepath.Join(release, cfg.Build.Binary) + if err = m.Operator.StartCandidate(ctx, candidateUnit, binary, cfg.Service.EnvironmentFile, env); err != nil { + return err + } + stopCandidate := true + defer func() { + if !stopCandidate { + return + } + stopCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = m.Operator.Stop(stopCtx, candidateUnit) + }() + probeLocal := m.probeHealthReadiness + if checkMarkers { + probeLocal = m.probeAll + } + if err = probeLocal(ctx, cfg, single.CandidateAddress); err != nil { + return fmt.Errorf("candidate failed: %w", err) + } + + oldHandler, err := os.ReadFile(single.CaddyHandler) + if err != nil { + return fmt.Errorf("read current Caddy handler: %w", err) + } + oldCurrent, err := resolveReleaseLink(cfg.Deployment.Root, single.CurrentLink) + if err != nil { + return err + } + oldPrevious, previousErr := resolveReleaseLink(cfg.Deployment.Root, single.PreviousLink) + handlerChanged := false + currentChanged := false + previousChanged := false + defer func() { + if err == nil { + return + } + recoveryErr := error(nil) + if currentChanged { + if restoreErr := replaceSymlink(single.CurrentLink, oldCurrent); restoreErr != nil { + recoveryErr = errors.Join(recoveryErr, restoreErr) + } else if restoreErr = m.Operator.Restart(ctx, single.Unit); restoreErr != nil { + recoveryErr = errors.Join(recoveryErr, restoreErr) + } else if restoreErr = m.probeHealthReadiness(ctx, cfg, single.Address); restoreErr != nil { + recoveryErr = errors.Join(recoveryErr, restoreErr) + } + } + if previousChanged { + var restoreErr error + if previousErr == nil { + restoreErr = replaceSymlink(single.PreviousLink, oldPrevious) + } else { + restoreErr = removeSymlink(single.PreviousLink) + } + recoveryErr = errors.Join(recoveryErr, restoreErr) + } + if handlerChanged && recoveryErr == nil { + if restoreErr := atomicWrite(single.CaddyHandler, oldHandler, 0o644); restoreErr != nil { + recoveryErr = errors.Join(recoveryErr, restoreErr) + } else if restoreErr = m.Operator.ValidateCaddy(ctx, single.CaddyConfig); restoreErr != nil { + recoveryErr = errors.Join(recoveryErr, restoreErr) + } else if restoreErr = m.Operator.ReloadCaddy(ctx); restoreErr != nil { + recoveryErr = errors.Join(recoveryErr, restoreErr) + } + } + if recoveryErr != nil && handlerChanged { + stopCandidate = false + err = &retainedCandidateError{cause: errors.Join(err, fmt.Errorf("singleton recovery incomplete; candidate remains routed for operator recovery: %w", recoveryErr))} + } + }() + + candidateHandler, err := renderHandler(single.CaddyHandlerTemplate, single.CandidateAddress) + if err != nil { + return err + } + if err = atomicWrite(single.CaddyHandler, candidateHandler, 0o644); err != nil { + return err + } + handlerChanged = true + if err = m.Operator.ValidateCaddy(ctx, single.CaddyConfig); err != nil { + return fmt.Errorf("candidate Caddy validation failed: %w", err) + } + if err = m.Operator.ReloadCaddy(ctx); err != nil { + return fmt.Errorf("candidate Caddy reload failed: %w", err) + } + if err = m.probePublic(ctx, cfg, checkMarkers); err != nil { + return fmt.Errorf("candidate public-origin smoke failed: %w", err) + } + + if err = replaceSymlink(single.PreviousLink, oldCurrent); err != nil { + return err + } + previousChanged = true + if err = replaceSymlink(single.CurrentLink, release); err != nil { + return err + } + currentChanged = true + if err = m.Operator.Restart(ctx, single.Unit); err != nil { + return err + } + if err = probeLocal(ctx, cfg, single.Address); err != nil { + return fmt.Errorf("post-activation smoke failed: %w", err) + } + installedHandler, err := renderHandler(single.CaddyHandlerTemplate, single.Address) + if err != nil { + return err + } + if err = atomicWrite(single.CaddyHandler, installedHandler, 0o644); err != nil { + return err + } + if err = m.Operator.ValidateCaddy(ctx, single.CaddyConfig); err != nil { + return fmt.Errorf("installed Caddy validation failed: %w", err) + } + if err = m.Operator.ReloadCaddy(ctx); err != nil { + return fmt.Errorf("installed Caddy reload failed: %w", err) + } + if err = m.probePublic(ctx, cfg, checkMarkers); err != nil { + return fmt.Errorf("public-origin smoke failed: %w", err) + } + if err = m.continuityWindow(ctx, cfg, single.CandidateAddress, checkMarkers); err != nil { + return fmt.Errorf("activation continuity failed: %w", err) + } + return nil +} + +func (m Manager) Status(ctx context.Context, cfg config.Config) (Status, error) { + result := Status{Units: map[string]bool{}} + record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if err == nil { + result.State = &record + result.StateInitialized = true + } else if !os.IsNotExist(err) { + return Status{}, err + } + units := []string{} + if cfg.Deployment.Strategy == "blue_green" { + units = []string{cfg.Deployment.BlueGreen.Blue.Unit, cfg.Deployment.BlueGreen.Green.Unit} + } else { + units = []string{cfg.Deployment.Singleton.Unit} + lease, exists, leaseErr := loadCandidateLease(cfg) + if leaseErr != nil { + return Status{}, leaseErr + } + if exists { + units = append(units, lease.Unit) + } else if result.StateInitialized && record.CandidateRelease != "" { + return Status{}, errors.New("candidate release has no operation-scoped lease; run tend reconcile --json") + } + } + for _, unit := range units { + active, err := m.Operator.IsActive(ctx, unit) + if err != nil { + return Status{}, err + } + result.Units[unit] = active + } + return result, nil +} + +func (m Manager) Prune(cfg config.Config, keep int, apply bool) ([]string, error) { + if keep < 2 || keep > 100 { + return nil, errors.New("keep must be between 2 and 100") + } + lock, err := acquireLock(cfg.Deployment.LockFile) + if err != nil { + return nil, err + } + defer lock.Close() + record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if err != nil { + return nil, err + } + entries, err := os.ReadDir(filepath.Join(cfg.Deployment.Root, "releases")) + if err != nil { + return nil, err + } + type candidate struct { + name, path string + mod time.Time + } + items := []candidate{} + protected := map[string]bool{record.ActiveRelease: true, record.PreviousRelease: true} + if cfg.Deployment.Strategy == "singleton_candidate" { + lease, exists, leaseErr := loadCandidateLease(cfg) + if leaseErr != nil { + return nil, leaseErr + } + if exists { + protected[lease.Release] = true + } else if record.CandidateRelease != "" { + protected[record.CandidateRelease] = true + } + } + for _, entry := range entries { + if !entry.IsDir() || entry.Type()&os.ModeSymlink != 0 || !strings.HasPrefix(entry.Name(), "sha256-") { + continue + } + path := filepath.Join(cfg.Deployment.Root, "releases", entry.Name()) + if protected[path] { + continue + } + info, err := entry.Info() + if err != nil { + return nil, err + } + items = append(items, candidate{entry.Name(), path, info.ModTime()}) + } + sort.Slice(items, func(i, j int) bool { return items[i].mod.After(items[j].mod) }) + retained := keep - 2 + if retained < 0 { + retained = 0 + } + if retained > len(items) { + retained = len(items) + } + items = items[retained:] + paths := make([]string, 0, len(items)) + for _, item := range items { + paths = append(paths, item.path) + if apply { + if err := removeRelease(item.path, cfg.Deployment.Root); err != nil { + return paths, err + } + } + } + return paths, nil +} + +func (m Manager) probeAll(ctx context.Context, cfg config.Config, address string) error { + checks := append([]config.Smoke{{Path: cfg.Deployment.HealthPath}, {Path: cfg.Deployment.ReadinessPath}}, cfg.Deployment.Smoke...) + return m.probe(ctx, cfg, address, checks) +} + +func (m Manager) probeHealthReadiness(ctx context.Context, cfg config.Config, address string) error { + checks := []config.Smoke{{Path: cfg.Deployment.HealthPath}, {Path: cfg.Deployment.ReadinessPath}} + return m.probe(ctx, cfg, address, checks) +} + +func (m Manager) probePublic(ctx context.Context, cfg config.Config, checkMarkers bool) error { + timeout := time.Duration(cfg.Deployment.CandidateTimeoutSecs) * time.Second + for _, check := range cfg.Deployment.PublicSmoke { + attempt, cancel := context.WithTimeout(ctx, timeout) + contains := check.Contains + if !checkMarkers { + contains = "" + } + err := m.Operator.ProbeURL(attempt, check.URL, contains) + cancel() + if err != nil { + return err + } + } + return nil +} + +func (m Manager) continuityWindow(ctx context.Context, cfg config.Config, previousAddress string, checkMarkers bool) error { + steps := cfg.Deployment.ActivationWindowSecs * 4 + if steps < 1 { + steps = 1 + } + for step := 0; step < steps; step++ { + if err := m.probePublic(ctx, cfg, checkMarkers); err != nil { + return err + } + if previousAddress != "" { + if err := m.probeHealthReadiness(ctx, cfg, previousAddress); err != nil { + return fmt.Errorf("previous slot lost continuity: %w", err) + } + } + if step+1 < steps { + sleep := m.Sleep + if sleep == nil { + sleep = sleepContext + } + if err := sleep(ctx, 250*time.Millisecond); err != nil { + return err + } + } + } + return nil +} + +func (m Manager) probe(ctx context.Context, cfg config.Config, address string, checks []config.Smoke) error { + timeout := time.Duration(cfg.Deployment.CandidateTimeoutSecs) * time.Second + for _, check := range checks { + deadline := m.Now().Add(timeout) + var last error + for { + attempt, cancel := context.WithTimeout(ctx, 2*time.Second) + last = m.Operator.Probe(attempt, address, cfg.Service.AllowedHost, check.Path, check.Contains) + cancel() + if last == nil { + break + } + if !m.Now().Before(deadline) { + return last + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(200 * time.Millisecond): + } + } + } + return nil +} +func slotConfig(bg config.BlueGreen, name string) config.Slot { + if name == "blue" { + return bg.Blue + } + return bg.Green +} +func resolveReleaseLink(root, link string) (string, error) { + info, err := os.Lstat(link) + if err != nil { + return "", err + } + if info.Mode()&os.ModeSymlink == 0 { + return "", errors.New("release pointer is not a symlink") + } + target, err := os.Readlink(link) + if err != nil { + return "", err + } + if !filepath.IsAbs(target) { + target = filepath.Join(filepath.Dir(link), target) + } + target = filepath.Clean(target) + probe := state.Record{SchemaVersion: state.SchemaVersion, Strategy: "singleton_candidate", ActiveSlot: "singleton", ActiveRelease: target, UpdatedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)} + if err := probe.Validate(root, "singleton_candidate"); err != nil { + return "", err + } + targetInfo, err := os.Lstat(target) + if err != nil { + return "", err + } + if !targetInfo.IsDir() || targetInfo.Mode()&os.ModeSymlink != 0 { + return "", errors.New("release target must be a real directory") + } + return target, nil +} +func replaceSymlink(link, target string) error { + if info, err := os.Lstat(link); err == nil && info.Mode()&os.ModeSymlink == 0 { + return errors.New("refusing to replace non-symlink release pointer") + } else if err != nil && !os.IsNotExist(err) { + return err + } + if err := os.MkdirAll(filepath.Dir(link), 0o755); err != nil { + return err + } + stage, err := os.MkdirTemp(filepath.Dir(link), ".tend-link-") + if err != nil { + return err + } + defer os.RemoveAll(stage) + tmp := filepath.Join(stage, "next") + if err := os.Symlink(target, tmp); err != nil { + return err + } + return os.Rename(tmp, link) +} +func removeSymlink(path string) error { + info, err := os.Lstat(path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink == 0 { + return errors.New("refusing to remove non-symlink") + } + return os.Remove(path) +} +func renderHandler(templatePath, address string) ([]byte, error) { + b, err := os.ReadFile(templatePath) + if err != nil { + return nil, err + } + const marker = "{{UPSTREAM}}" + if bytes := strings.Count(string(b), marker); bytes != 1 { + return nil, errors.New("Caddy handler template must contain exactly one upstream marker") + } + return []byte(strings.Replace(string(b), marker, address, 1)), nil +} +func atomicWrite(path string, data []byte, mode os.FileMode) error { + var existing os.FileInfo + if info, err := os.Lstat(path); err == nil { + if info.Mode()&os.ModeSymlink != 0 || !info.Mode().IsRegular() { + return errors.New("refusing to replace non-regular or symlink file") + } + existing = info + } else if !os.IsNotExist(err) { + return err + } + identity := identityFor(existing, mode) + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, ".tend-write-") + if err != nil { + return err + } + name := tmp.Name() + ok := false + defer func() { + _ = tmp.Close() + if !ok { + _ = os.Remove(name) + } + }() + if err := applyIdentity(tmp, identity); err != nil { + return err + } + if _, err := tmp.Write(data); err != nil { + return err + } + if err := tmp.Sync(); err != nil { + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(name, path); err != nil { + return err + } + ok = true + return syncDirectory(dir) +} +func syncDirectory(path string) error { + dir, err := os.Open(path) + if err != nil { + return err + } + defer dir.Close() + return dir.Sync() +} +func removeRelease(path, root string) error { + releases := filepath.Join(root, "releases") + rel, err := filepath.Rel(releases, path) + if err != nil || rel == "." || rel == ".." || strings.ContainsRune(rel, filepath.Separator) || !strings.HasPrefix(rel, "sha256-") { + return errors.New("unsafe prune target") + } + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("prune target is not a real release directory") + } + return os.RemoveAll(path) +} +func Marshal(value any) ([]byte, error) { + b, err := json.MarshalIndent(value, "", " ") + if err != nil { + return nil, err + } + return append(b, '\n'), nil +} diff --git a/internal/deploy/deploy_test.go b/internal/deploy/deploy_test.go new file mode 100644 index 0000000..95bfff0 --- /dev/null +++ b/internal/deploy/deploy_test.go @@ -0,0 +1,591 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package deploy + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gamertan.com/tend/internal/config" + "gamertan.com/tend/internal/eventlog" + "gamertan.com/tend/internal/state" +) + +const testCandidateUnit = "example-site-tend-candidate-dddddddddddd.service" + +func storeTestCandidateLease(t *testing.T, cfg config.Config, release, startedAt string) { + t.Helper() + lease := state.CandidateLease{SchemaVersion: state.CandidateLeaseSchemaVersion, Service: cfg.Service.Name, OperationID: strings.Repeat("d", 32), Release: release, Unit: testCandidateUnit, Address: cfg.Deployment.Singleton.CandidateAddress, StartedAt: startedAt} + if err := state.StoreCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, lease); err != nil { + t.Fatal(err) + } +} + +type fakeOperator struct { + failReload bool + failReloadAt int + reloads int + failRestartUnit string + failRestartOnce bool + rejectMarkers bool + active map[string]bool + starts, stops, restarts []string + probes []string + publicProbes []string + failPublic bool + failPublicAfter int + candidateEnvironment map[string]string + candidateFile string +} + +func (f *fakeOperator) Restart(_ context.Context, unit string) error { + f.restarts = append(f.restarts, unit) + if unit == f.failRestartUnit { + if f.failRestartOnce { + f.failRestartUnit = "" + } + return errors.New("injected restart failure") + } + f.active[unit] = true + return nil +} +func (f *fakeOperator) Stop(_ context.Context, unit string) error { + f.stops = append(f.stops, unit) + f.active[unit] = false + return nil +} +func (f *fakeOperator) IsActive(_ context.Context, unit string) (bool, error) { + return f.active[unit], nil +} +func (f *fakeOperator) StartCandidate(_ context.Context, unit, binary, environmentFile string, env map[string]string) error { + if !filepath.IsAbs(binary) || !filepath.IsAbs(environmentFile) || len(env) == 0 { + return errors.New("bad candidate") + } + f.starts = append(f.starts, unit) + f.candidateFile = environmentFile + f.candidateEnvironment = make(map[string]string, len(env)) + for key, value := range env { + f.candidateEnvironment[key] = value + } + f.active[unit] = true + return nil +} +func (f *fakeOperator) ProbeURL(_ context.Context, value, contains string) error { + f.publicProbes = append(f.publicProbes, value) + if f.failPublic || (f.failPublicAfter > 0 && len(f.publicProbes) >= f.failPublicAfter) { + return errors.New("injected public smoke failure") + } + if f.rejectMarkers && contains != "" { + return errors.New("unexpected future-release smoke marker") + } + return nil +} + +func TestPublicSmokeFailureRestoresBlueGreenHandlerAndSlot(t *testing.T) { + cfg, old, fresh := baseConfig(t, "blue_green") + handler := filepath.Join(cfg.Deployment.Root, "handler.caddy") + template := filepath.Join(cfg.Deployment.Root, "handler.template") + original := []byte("reverse_proxy 127.0.0.1:8090\n") + _ = os.WriteFile(handler, original, 0o644) + _ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644) + blue := filepath.Join(cfg.Deployment.Root, "slots", "blue") + green := filepath.Join(cfg.Deployment.Root, "slots", "green") + _ = replaceSymlink(blue, old) + _ = replaceSymlink(green, old) + cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}} + operator := &fakeOperator{active: map[string]bool{}, failPublic: true} + if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true}); err == nil { + t.Fatal("expected public smoke failure") + } + body, _ := os.ReadFile(handler) + if string(body) != string(original) { + t.Fatalf("handler not restored: %q", body) + } + target, err := resolveReleaseLink(cfg.Deployment.Root, green) + if err != nil || target != old { + t.Fatalf("green=%q err=%v", target, err) + } + record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if err != nil { + t.Fatal(err) + } + if record.ActiveRelease != old || record.LastAttemptOutcome != "failed" || record.CandidateRelease != "" || record.LastAttemptRelease != fresh { + t.Fatalf("failed attempt state=%+v", record) + } +} + +func TestContinuityFailureRestoresBlueGreenRoute(t *testing.T) { + cfg, old, fresh := baseConfig(t, "blue_green") + handler := filepath.Join(cfg.Deployment.Root, "handler.caddy") + template := filepath.Join(cfg.Deployment.Root, "handler.template") + original := []byte("reverse_proxy 127.0.0.1:8090\n") + _ = os.WriteFile(handler, original, 0o644) + _ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644) + blue := filepath.Join(cfg.Deployment.Root, "slots", "blue") + green := filepath.Join(cfg.Deployment.Root, "slots", "green") + _ = replaceSymlink(blue, old) + _ = replaceSymlink(green, old) + cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}} + operator := &fakeOperator{active: map[string]bool{}, failPublicAfter: 3} + if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err == nil || !strings.Contains(err.Error(), "continuity") { + t.Fatalf("expected continuity failure, got %v", err) + } + body, _ := os.ReadFile(handler) + if string(body) != string(original) { + t.Fatalf("handler not restored: %q", body) + } + target, err := resolveReleaseLink(cfg.Deployment.Root, green) + if err != nil || target != old { + t.Fatalf("green=%q err=%v", target, err) + } +} +func (f *fakeOperator) ValidateCaddy(context.Context, string) error { return nil } +func (f *fakeOperator) ReloadCaddy(context.Context) error { + f.reloads++ + if f.failReload || (f.failReloadAt > 0 && f.reloads == f.failReloadAt) { + return errors.New("injected reload failure") + } + return nil +} +func (f *fakeOperator) Probe(_ context.Context, address, host, path, contains string) error { + f.probes = append(f.probes, address+path) + if f.rejectMarkers && contains != "" { + return errors.New("unexpected future-release smoke marker") + } + return nil +} + +func baseConfig(t *testing.T, strategy string) (config.Config, string, string) { + t.Helper() + root := filepath.Join(t.TempDir(), "service") + if err := os.MkdirAll(filepath.Join(root, "releases"), 0o755); err != nil { + t.Fatal(err) + } + old := filepath.Join(root, "releases", "sha256-"+strings.Repeat("c", 64)) + fresh := filepath.Join(root, "releases", "sha256-"+strings.Repeat("a", 64)) + for _, dir := range []string{old, fresh} { + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "app"), []byte("x"), 0o755); err != nil { + t.Fatal(err) + } + } + cfg := config.Config{SchemaVersion: 2, Service: config.Service{Name: "example-site", AllowedHost: "example.test", EnvironmentFile: "/etc/tend/environment/example-site.env"}, Build: config.Build{Package: "./cmd/site", Binary: "app", Branch: "main"}, Deployment: config.Deployment{Strategy: strategy, Root: root, LockFile: filepath.Join(root, "deploy.lock"), StateFile: filepath.Join(root, "state.json"), EventLog: filepath.Join(root, "deployment-events.jsonl"), HealthPath: "/healthz", ReadinessPath: "/readyz", CandidateTimeoutSecs: 2, ActivationWindowSecs: 1, Smoke: []config.Smoke{{Path: "/", Contains: "Example"}}, PublicSmoke: []config.PublicSmoke{{URL: "https://example.test/", Contains: "Example"}}}} + return cfg, old, fresh +} +func manager(operator Operator, fresh string) Manager { + return Manager{Operator: operator, Now: func() time.Time { return time.Unix(100, 0).UTC() }, Prepare: func(config.Config, string, string, string) (string, error) { return fresh, nil }, Inspect: func(config.Config, string, string, string) error { return nil }, ReadIdentity: func(string) (releaseIdentity, error) { + return releaseIdentity{Version: "v0.2.0-preview.1", Commit: strings.Repeat("b", 40)}, nil + }, OperationID: func() (string, error) { return strings.Repeat("d", 32), nil }, Sleep: func(context.Context, time.Duration) error { return nil }} +} + +func singletonSettings(t *testing.T, cfg config.Config, currentRelease string) (*config.Singleton, string) { + t.Helper() + handler := filepath.Join(cfg.Deployment.Root, "handler.caddy") + template := filepath.Join(cfg.Deployment.Root, "handler.template") + if err := os.WriteFile(handler, []byte("reverse_proxy 127.0.0.1:8092\n"), 0o640); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644); err != nil { + t.Fatal(err) + } + current := filepath.Join(cfg.Deployment.Root, "current") + if err := replaceSymlink(current, currentRelease); err != nil { + t.Fatal(err) + } + return &config.Singleton{ + Unit: "example-site.service", Address: "127.0.0.1:8092", CandidateAddress: "127.0.0.1:18092", ListenEnv: "EXAMPLE_LISTEN", + CurrentLink: current, PreviousLink: filepath.Join(cfg.Deployment.Root, "previous"), CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), + CaddyHandler: handler, CaddyHandlerTemplate: template, + }, handler +} + +func TestBlueGreenActivationAndRollback(t *testing.T) { + cfg, old, fresh := baseConfig(t, "blue_green") + handler := filepath.Join(cfg.Deployment.Root, "handler.caddy") + template := filepath.Join(cfg.Deployment.Root, "handler.template") + if err := os.WriteFile(handler, []byte("reverse_proxy 127.0.0.1:8090\n"), 0o640); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644); err != nil { + t.Fatal(err) + } + blue := filepath.Join(cfg.Deployment.Root, "slots", "blue") + green := filepath.Join(cfg.Deployment.Root, "slots", "green") + if err := replaceSymlink(blue, old); err != nil { + t.Fatal(err) + } + if err := replaceSymlink(green, old); err != nil { + t.Fatal(err) + } + cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}} + if err := cfg.Validate(); err != nil { + t.Fatal(err) + } + operator := &fakeOperator{active: map[string]bool{"example-blue.service": true, "example-green.service": true}} + m := manager(operator, fresh) + var events []eventlog.Event + m.AppendEvent = func(_ string, event eventlog.Event) error { + events = append(events, event) + return nil + } + report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}) + if err != nil { + t.Fatal(err) + } + if report.ActiveRelease != fresh || report.PreviousRelease != old { + t.Fatalf("report=%+v", report) + } + deployed, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if err != nil { + t.Fatal(err) + } + if deployed.DesiredRelease != fresh || deployed.CandidateRelease != "" || deployed.LastAttemptRelease != fresh || deployed.LastAttemptOutcome != "succeeded" { + t.Fatalf("deployment identity state=%+v", deployed) + } + if len(events) != 2 || events[0].Phase != "candidate" || events[0].Outcome != "running" || events[1].Phase != "activation" || events[1].Outcome != "succeeded" || events[0].OperationID != events[1].OperationID { + t.Fatalf("events=%+v", events) + } + if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 { + t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err) + } + if target, err := resolveReleaseLink(cfg.Deployment.Root, green); err != nil || target != fresh { + t.Fatalf("green=%q err=%v", target, err) + } + operator.rejectMarkers = true + operator.probes = nil + record, err := m.Rollback(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + if record.ActiveRelease != old || record.PreviousRelease != fresh { + t.Fatalf("rollback=%+v", record) + } + if len(events) != 4 || events[2].Phase != "rollback" || events[2].Outcome != "running" || events[3].Phase != "rollback" || events[3].Outcome != "succeeded" || events[2].OperationID != events[3].OperationID || events[2].ArtifactDigest != strings.Repeat("c", 64) { + t.Fatalf("rollback events=%+v", events) + } + if len(operator.probes) != 4 { + t.Fatalf("rollback probes=%#v", operator.probes) + } + for _, probe := range operator.probes { + if !strings.HasSuffix(probe, "/healthz") && !strings.HasSuffix(probe, "/readyz") { + t.Fatalf("rollback applied future-release smoke checks: %#v", operator.probes) + } + } +} + +func TestDeploymentEvidenceCanNeverBlockActivationOrRollback(t *testing.T) { + cfg, old, fresh := baseConfig(t, "singleton_candidate") + cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old) + m := manager(&fakeOperator{active: map[string]bool{}}, fresh) + appendCalls := 0 + m.AppendEvent = func(string, eventlog.Event) error { + appendCalls++ + return errors.New("injected event failure") + } + report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}) + if err != nil { + t.Fatal(err) + } + if report.EventWarnings != 2 || report.ActiveRelease != fresh { + t.Fatalf("report=%+v", report) + } + if _, err := m.Rollback(context.Background(), cfg); err != nil { + t.Fatal(err) + } + record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if err != nil || record.ActiveRelease != old || appendCalls != 4 { + t.Fatalf("record=%+v appends=%d err=%v", record, appendCalls, err) + } +} + +func TestDeploymentEvidenceIdentityIsBestEffort(t *testing.T) { + cfg, old, fresh := baseConfig(t, "singleton_candidate") + cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old) + m := manager(&fakeOperator{active: map[string]bool{}}, fresh) + m.ReadIdentity = func(string) (releaseIdentity, error) { + return releaseIdentity{}, errors.New("injected identity failure") + } + report, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}) + if err != nil || report.EventWarnings != 1 || report.ActiveRelease != fresh { + t.Fatalf("report=%+v err=%v", report, err) + } +} + +func TestSingletonOperationIdentityIsRequiredBeforeCandidateStart(t *testing.T) { + cfg, old, fresh := baseConfig(t, "singleton_candidate") + cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old) + operator := &fakeOperator{active: map[string]bool{}} + m := manager(operator, fresh) + m.OperationID = func() (string, error) { return "", errors.New("injected entropy failure") } + if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err == nil || !strings.Contains(err.Error(), "operation identity") { + t.Fatalf("expected operation identity refusal, got %v", err) + } + if len(operator.starts) != 0 { + t.Fatalf("candidate started without an operation identity: %#v", operator.starts) + } +} + +func TestSingletonUnresolvedLeaseBlocksReplacementBeforeCandidateStart(t *testing.T) { + cfg, old, fresh := baseConfig(t, "singleton_candidate") + cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old) + at := time.Unix(100, 0).UTC().Format(time.RFC3339) + record := state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: fresh, CandidateRelease: fresh, ActiveSlot: "singleton", ActiveRelease: old, LastAttemptRelease: fresh, LastAttemptOutcome: "failed", LastAttemptAt: at, UpdatedAt: at} + if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil { + t.Fatal(err) + } + storeTestCandidateLease(t, cfg, fresh, at) + operator := &fakeOperator{active: map[string]bool{testCandidateUnit: true}} + if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err == nil || !strings.Contains(err.Error(), "run tend reconcile") { + t.Fatalf("expected unresolved lease refusal, got %v", err) + } + if len(operator.starts) != 0 || !operator.active[testCandidateUnit] { + t.Fatalf("existing candidate was disturbed: starts=%#v active=%#v", operator.starts, operator.active) + } +} + +func TestBlueGreenCaddyFailureRestoresHandlerAndSlot(t *testing.T) { + cfg, old, fresh := baseConfig(t, "blue_green") + handler := filepath.Join(cfg.Deployment.Root, "handler.caddy") + template := filepath.Join(cfg.Deployment.Root, "handler.template") + original := []byte("reverse_proxy 127.0.0.1:8090\n") + _ = os.WriteFile(handler, original, 0o644) + _ = os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644) + blue := filepath.Join(cfg.Deployment.Root, "slots", "blue") + green := filepath.Join(cfg.Deployment.Root, "slots", "green") + _ = replaceSymlink(blue, old) + _ = replaceSymlink(green, old) + cfg.Deployment.BlueGreen = &config.BlueGreen{CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, CaddyHandlerTemplate: template, BootstrapActive: "blue", Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}} + operator := &fakeOperator{active: map[string]bool{}, failReload: true} + m := manager(operator, fresh) + if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil { + t.Fatal("expected failure") + } + body, _ := os.ReadFile(handler) + if string(body) != string(original) { + t.Fatalf("handler not restored: %q", body) + } + target, err := resolveReleaseLink(cfg.Deployment.Root, green) + if err != nil || target != old { + t.Fatalf("green=%q err=%v", target, err) + } + failed, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if err != nil { + t.Fatal(err) + } + if failed.ActiveRelease != old || failed.LastAttemptOutcome != "failed" || failed.CandidateRelease != "" { + t.Fatalf("failed state=%+v", failed) + } +} + +func TestSingletonRestartFailureRestoresPointers(t *testing.T) { + cfg, old, fresh := baseConfig(t, "singleton_candidate") + settings, handler := singletonSettings(t, cfg, old) + cfg.Deployment.Singleton = settings + operator := &fakeOperator{active: map[string]bool{}, failRestartUnit: "example-site.service", failRestartOnce: true} + m := manager(operator, fresh) + if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil { + t.Fatal("expected failure") + } + target, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink) + if err != nil || target != old { + t.Fatalf("current=%q err=%v", target, err) + } + if _, err := os.Lstat(cfg.Deployment.Singleton.PreviousLink); !os.IsNotExist(err) { + t.Fatal("previous pointer was not restored") + } + body, err := os.ReadFile(handler) + if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" { + t.Fatalf("handler=%q err=%v", body, err) + } + if info, err := os.Stat(handler); err != nil || info.Mode().Perm() != 0o640 { + t.Fatalf("handler mode=%v err=%v", info.Mode().Perm(), err) + } + if operator.active[testCandidateUnit] { + t.Fatal("candidate was not stopped after successful restoration") + } +} + +func TestStateRecordsSuccessfulActivation(t *testing.T) { + cfg, old, fresh := baseConfig(t, "singleton_candidate") + settings, handler := singletonSettings(t, cfg, old) + cfg.Deployment.Singleton = settings + operator := &fakeOperator{active: map[string]bool{}} + m := manager(operator, fresh) + if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err != nil { + t.Fatal(err) + } + record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if err != nil { + t.Fatal(err) + } + if record.ActiveRelease != fresh || record.PreviousRelease != old || record.CandidateRelease != "" { + t.Fatalf("state=%+v", record) + } + if _, err := os.Lstat(state.CandidateLeasePath(cfg.Deployment.StateFile)); !os.IsNotExist(err) { + t.Fatalf("candidate lease was not removed: %v", err) + } + if len(operator.starts) != 1 || operator.starts[0] != testCandidateUnit { + t.Fatalf("candidate starts=%#v", operator.starts) + } + if operator.candidateFile != cfg.Service.EnvironmentFile || len(operator.candidateEnvironment) != 1 || operator.candidateEnvironment["EXAMPLE_LISTEN"] != "127.0.0.1:18092" { + t.Fatalf("candidate file=%q environment=%#v", operator.candidateFile, operator.candidateEnvironment) + } + body, err := os.ReadFile(handler) + if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" { + t.Fatalf("handler=%q err=%v", body, err) + } + if operator.reloads != 2 || operator.active[testCandidateUnit] { + t.Fatalf("reloads=%d active=%#v", operator.reloads, operator.active) + } + reconciliation, err := m.Reconcile(context.Background(), cfg) + if err != nil || reconciliation.Mutation != "none" || reconciliation.Disposition != "settled" || reconciliation.Observed.CandidateLease || !reconciliation.Consistent { + t.Fatalf("reconciliation=%+v err=%v", reconciliation, err) + } +} + +func TestSingletonContinuityFailureRestoresHandlerPointersAndService(t *testing.T) { + cfg, old, fresh := baseConfig(t, "singleton_candidate") + settings, handler := singletonSettings(t, cfg, old) + cfg.Deployment.Singleton = settings + operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failPublicAfter: 4} + m := manager(operator, fresh) + if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true}); err == nil || !strings.Contains(err.Error(), "continuity") { + t.Fatalf("expected continuity failure, got %v", err) + } + current, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink) + if err != nil || current != old { + t.Fatalf("current=%q err=%v", current, err) + } + body, err := os.ReadFile(handler) + if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" { + t.Fatalf("handler=%q err=%v", body, err) + } + if operator.active[testCandidateUnit] { + t.Fatal("candidate was not stopped after continuity restoration") + } +} + +func TestSingletonCaddyReloadFailuresRestorePriorRoute(t *testing.T) { + for _, reload := range []int{1, 2} { + t.Run(fmt.Sprintf("reload-%d", reload), func(t *testing.T) { + cfg, old, fresh := baseConfig(t, "singleton_candidate") + settings, handler := singletonSettings(t, cfg, old) + cfg.Deployment.Singleton = settings + operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failReloadAt: reload} + if _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true}); err == nil || !strings.Contains(err.Error(), "Caddy reload failed") { + t.Fatalf("expected Caddy reload failure, got %v", err) + } + current, err := resolveReleaseLink(cfg.Deployment.Root, settings.CurrentLink) + if err != nil || current != old { + t.Fatalf("current=%q err=%v", current, err) + } + body, err := os.ReadFile(handler) + if err != nil || string(body) != "reverse_proxy 127.0.0.1:8092\n" { + t.Fatalf("handler=%q err=%v", body, err) + } + if operator.active[testCandidateUnit] { + t.Fatal("candidate was not stopped after route restoration") + } + }) + } +} + +func TestSingletonIncompleteRecoveryKeepsProvenCandidateRunning(t *testing.T) { + cfg, old, fresh := baseConfig(t, "singleton_candidate") + settings, _ := singletonSettings(t, cfg, old) + cfg.Deployment.Singleton = settings + operator := &fakeOperator{active: map[string]bool{"example-site.service": true}, failReload: true} + _, err := manager(operator, fresh).Deploy(context.Background(), cfg, Request{Activate: true}) + if err == nil || !strings.Contains(err.Error(), "candidate remains routed for operator recovery") { + t.Fatalf("expected explicit incomplete recovery, got %v", err) + } + if !operator.active[testCandidateUnit] { + t.Fatal("proven candidate was stopped despite incomplete route restoration") + } + record, loadErr := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + lease, leaseErr := state.LoadCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, cfg.Service.Name) + if loadErr != nil || leaseErr != nil || record.CandidateRelease != fresh || lease.OperationID != strings.Repeat("d", 32) || lease.Unit != testCandidateUnit { + t.Fatalf("retained state=%+v err=%v", record, loadErr) + } +} + +func TestReconcileReportsRetainedCandidateWithoutMutation(t *testing.T) { + cfg, old, fresh := baseConfig(t, "singleton_candidate") + settings, handler := singletonSettings(t, cfg, old) + cfg.Deployment.Singleton = settings + at := time.Unix(100, 0).UTC().Format(time.RFC3339) + record := state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: fresh, CandidateRelease: fresh, ActiveSlot: "singleton", ActiveRelease: old, LastAttemptRelease: fresh, LastAttemptOutcome: "failed", LastAttemptAt: at, UpdatedAt: at} + if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil { + t.Fatal(err) + } + storeTestCandidateLease(t, cfg, fresh, at) + candidateHandler, err := renderHandler(settings.CaddyHandlerTemplate, settings.CandidateAddress) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(handler, candidateHandler, 0o640); err != nil { + t.Fatal(err) + } + operator := &fakeOperator{active: map[string]bool{settings.Unit: true, testCandidateUnit: true}} + reconciliation, err := manager(operator, fresh).Reconcile(context.Background(), cfg) + if err != nil || reconciliation.Mutation != "none" || !reconciliation.Observed.CandidateLease || reconciliation.Observed.CandidateUnitActive == nil || !*reconciliation.Observed.CandidateUnitActive || reconciliation.Observed.HandlerFileTarget != "candidate" || reconciliation.Disposition != "retained_candidate_handler_file" { + t.Fatalf("reconciliation=%+v err=%v", reconciliation, err) + } + if len(operator.stops) != 0 || len(operator.restarts) != 0 || operator.reloads != 0 { + t.Fatalf("reconcile mutated services: stops=%#v restarts=%#v reloads=%d", operator.stops, operator.restarts, operator.reloads) + } +} + +func TestSingletonRollbackRetainsOperationLeaseWhenRecoveryIsIncomplete(t *testing.T) { + cfg, old, fresh := baseConfig(t, "singleton_candidate") + cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, old) + operator := &fakeOperator{active: map[string]bool{cfg.Deployment.Singleton.Unit: true}} + m := manager(operator, fresh) + if _, err := m.Deploy(context.Background(), cfg, Request{Activate: true, ApprovedSHA256: strings.Repeat("a", 64)}); err != nil { + t.Fatal(err) + } + operator.failReload = true + if _, err := m.Rollback(context.Background(), cfg); err == nil || !strings.Contains(err.Error(), "candidate remains routed for operator recovery") { + t.Fatalf("expected retained rollback candidate, got %v", err) + } + record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if err != nil { + t.Fatal(err) + } + lease, leaseErr := state.LoadCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, cfg.Service.Name) + if record.ActiveRelease != fresh || record.CandidateRelease != old || leaseErr != nil || lease.OperationID != strings.Repeat("d", 32) || lease.Unit != testCandidateUnit || record.LastAttemptOutcome != "failed" { + t.Fatalf("rollback state=%+v", record) + } + if !operator.active[testCandidateUnit] { + t.Fatal("rollback candidate was stopped despite incomplete recovery") + } +} + +func TestPruneProtectsOperationScopedCandidateRelease(t *testing.T) { + cfg, active, candidate := baseConfig(t, "singleton_candidate") + cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, active) + at := time.Unix(100, 0).UTC().Format(time.RFC3339) + record := state.Record{SchemaVersion: state.SchemaVersion, Strategy: cfg.Deployment.Strategy, DesiredRelease: candidate, CandidateRelease: candidate, ActiveSlot: "singleton", ActiveRelease: active, LastAttemptRelease: candidate, LastAttemptOutcome: "failed", LastAttemptAt: at, UpdatedAt: at} + if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil { + t.Fatal(err) + } + storeTestCandidateLease(t, cfg, candidate, at) + removed, err := manager(&fakeOperator{active: map[string]bool{}}, candidate).Prune(cfg, 2, true) + if err != nil { + t.Fatal(err) + } + if len(removed) != 0 { + t.Fatalf("candidate release was selected for pruning: %#v", removed) + } + if info, err := os.Stat(candidate); err != nil || !info.IsDir() { + t.Fatalf("candidate release was not preserved: %v", err) + } +} diff --git a/internal/deploy/lock_linux.go b/internal/deploy/lock_linux.go new file mode 100644 index 0000000..1e5b25c --- /dev/null +++ b/internal/deploy/lock_linux.go @@ -0,0 +1,32 @@ +//go:build linux + +// SPDX-License-Identifier: AGPL-3.0-only + +package deploy + +import ( + "errors" + "os" + "syscall" +) + +type fileLock struct{ file *os.File } + +func acquireLock(path string) (*fileLock, error) { + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, err + } + if err := syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + _ = file.Close() + return nil, errors.New("deployment lock is held") + } + return &fileLock{file: file}, nil +} +func (l *fileLock) Close() error { + if l == nil || l.file == nil { + return nil + } + _ = syscall.Flock(int(l.file.Fd()), syscall.LOCK_UN) + return l.file.Close() +} diff --git a/internal/deploy/lock_linux_test.go b/internal/deploy/lock_linux_test.go new file mode 100644 index 0000000..37657b6 --- /dev/null +++ b/internal/deploy/lock_linux_test.go @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: AGPL-3.0-only +//go:build linux + +package deploy + +import ( + "path/filepath" + "testing" +) + +func TestHostWideLockSerializesIndependentServices(t *testing.T) { + path := filepath.Join(t.TempDir(), "tend-deploy.lock") + first, err := acquireLock(path) + if err != nil { + t.Fatal(err) + } + defer first.Close() + if second, err := acquireLock(path); err == nil { + _ = second.Close() + t.Fatal("second service acquired the shared activation lock") + } + if err = first.Close(); err != nil { + t.Fatal(err) + } + third, err := acquireLock(path) + if err != nil { + t.Fatal(err) + } + _ = third.Close() +} diff --git a/internal/deploy/lock_other.go b/internal/deploy/lock_other.go new file mode 100644 index 0000000..1a3cb69 --- /dev/null +++ b/internal/deploy/lock_other.go @@ -0,0 +1,14 @@ +//go:build !linux + +// SPDX-License-Identifier: AGPL-3.0-only + +package deploy + +import "errors" + +type fileLock struct{} + +func acquireLock(string) (*fileLock, error) { + return nil, errors.New("deployment mutations require Linux") +} +func (*fileLock) Close() error { return nil } diff --git a/internal/deploy/operator.go b/internal/deploy/operator.go new file mode 100644 index 0000000..1e81a0d --- /dev/null +++ b/internal/deploy/operator.go @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package deploy + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + "time" + + "gamertan.com/tend/internal/process" +) + +type Operator interface { + Restart(context.Context, string) error + Stop(context.Context, string) error + IsActive(context.Context, string) (bool, error) + StartCandidate(context.Context, string, string, string, map[string]string) error + ValidateCaddy(context.Context, string) error + ReloadCaddy(context.Context) error + Probe(context.Context, string, string, string, string) error + ProbeURL(context.Context, string, string) error +} + +type SystemOperator struct { + Runner process.Runner + Timeout time.Duration +} + +func (o SystemOperator) Restart(ctx context.Context, unit string) error { + _, err := o.Runner.Run(ctx, "/", nil, "systemctl", "restart", unit) + return err +} +func (o SystemOperator) Stop(ctx context.Context, unit string) error { + _, err := o.Runner.Run(ctx, "/", nil, "systemctl", "stop", unit) + return err +} +func (o SystemOperator) IsActive(ctx context.Context, unit string) (bool, error) { + out, err := o.Runner.Run(ctx, "/", nil, "systemctl", "is-active", unit) + if err != nil { + if strings.TrimSpace(string(out)) == "inactive" || strings.TrimSpace(string(out)) == "failed" { + return false, nil + } + return false, err + } + return strings.TrimSpace(string(out)) == "active", nil +} +func (o SystemOperator) StartCandidate(ctx context.Context, unit, binary, environmentFile string, env map[string]string) error { + args := []string{ + "--unit", unit, "--collect", + "--property=DynamicUser=yes", "--property=NoNewPrivileges=yes", + "--property=PrivateDevices=yes", "--property=PrivateTmp=yes", + "--property=ProtectClock=yes", "--property=ProtectControlGroups=yes", + "--property=ProtectHome=yes", "--property=ProtectHostname=yes", + "--property=ProtectKernelLogs=yes", "--property=ProtectKernelModules=yes", + "--property=ProtectKernelTunables=yes", "--property=ProtectSystem=strict", + "--property=RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX", + "--property=RestrictNamespaces=yes", "--property=RestrictRealtime=yes", + "--property=RestrictSUIDSGID=yes", "--property=LockPersonality=yes", + "--property=MemoryDenyWriteExecute=yes", "--property=CapabilityBoundingSet=", + "--property=AmbientCapabilities=", + "--property=EnvironmentFile=" + environmentFile, + } + keys := make([]string, 0, len(env)) + for key := range env { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + args = append(args, "--setenv", key+"="+env[key]) + } + args = append(args, "--", binary) + _, err := o.Runner.Run(ctx, "/", nil, "systemd-run", args...) + return err +} +func (o SystemOperator) ValidateCaddy(ctx context.Context, path string) error { + _, err := o.Runner.Run(ctx, "/", nil, "caddy", "validate", "--config", path, "--adapter", "caddyfile") + return err +} +func (o SystemOperator) ReloadCaddy(ctx context.Context) error { + _, err := o.Runner.Run(ctx, "/", nil, "systemctl", "reload", "caddy.service") + return err +} +func (o SystemOperator) Probe(ctx context.Context, address, host, path, contains string) error { + u := url.URL{Scheme: "http", Host: address, Path: path} + return o.probeRequest(ctx, u.String(), host, contains) +} +func (o SystemOperator) ProbeURL(ctx context.Context, value, contains string) error { + u, err := url.Parse(value) + if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.Fragment != "" { + return errors.New("public probe URL is invalid") + } + return o.probeRequest(ctx, u.String(), "", contains) +} +func (o SystemOperator) probeRequest(ctx context.Context, value, host, contains string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, value, nil) + if err != nil { + return err + } + if host != "" { + req.Host = host + } + client := &http.Client{Timeout: o.Timeout, CheckRedirect: func(*http.Request, []*http.Request) error { return errors.New("redirect refused") }} + response, err := client.Do(req) + if err != nil { + return err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return fmt.Errorf("probe returned HTTP %d", response.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(response.Body, 1<<20)) + if err != nil { + return err + } + if contains != "" && !strings.Contains(string(body), contains) { + return errors.New("probe response omitted required marker") + } + return nil +} diff --git a/internal/deploy/operator_test.go b/internal/deploy/operator_test.go new file mode 100644 index 0000000..583f5e6 --- /dev/null +++ b/internal/deploy/operator_test.go @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package deploy + +import ( + "context" + "reflect" + "strings" + "testing" +) + +type recordingRunner struct { + name string + args []string +} + +func (r *recordingRunner) Run(_ context.Context, _ string, _ map[string]string, name string, args ...string) ([]byte, error) { + r.name = name + r.args = append([]string(nil), args...) + return nil, nil +} + +func TestStartCandidateUsesArgumentVectorAndHardenedUnit(t *testing.T) { + runner := &recordingRunner{} + operator := SystemOperator{Runner: runner} + env := map[string]string{"Z_ENV": "safe value", "A_ENV": "first"} + if err := operator.StartCandidate(context.Background(), "example-tend-candidate.service", "/opt/example/releases/sha256-a/app", "/etc/tend/environment/example.env", env); err != nil { + t.Fatal(err) + } + if runner.name != "systemd-run" { + t.Fatalf("command=%q", runner.name) + } + required := []string{"--property=DynamicUser=yes", "--property=NoNewPrivileges=yes", "--property=ProtectSystem=strict", "--property=MemoryDenyWriteExecute=yes", "--property=CapabilityBoundingSet=", "--property=EnvironmentFile=/etc/tend/environment/example.env", "--setenv", "A_ENV=first", "--setenv", "Z_ENV=safe value", "--", "/opt/example/releases/sha256-a/app"} + cursor := 0 + for _, arg := range runner.args { + if cursor < len(required) && arg == required[cursor] { + cursor++ + } + } + if cursor != len(required) { + t.Fatalf("arguments omitted ordered security boundary: %#v", runner.args) + } + if reflect.DeepEqual(runner.args, []string{"sh", "-c"}) { + t.Fatal("candidate command used a shell") + } + if strings.Contains(strings.Join(runner.args, "\n"), "SUPER_SECRET") { + t.Fatal("candidate arguments exposed a secret value") + } +} diff --git a/internal/deploy/ownership_linux.go b/internal/deploy/ownership_linux.go new file mode 100644 index 0000000..a09ac12 --- /dev/null +++ b/internal/deploy/ownership_linux.go @@ -0,0 +1,38 @@ +//go:build linux + +// SPDX-License-Identifier: AGPL-3.0-only + +package deploy + +import ( + "os" + "syscall" +) + +type fileIdentity struct { + mode os.FileMode + uid, gid int + owned bool +} + +func identityFor(info os.FileInfo, fallback os.FileMode) fileIdentity { + identity := fileIdentity{mode: fallback} + if info == nil { + return identity + } + identity.mode = info.Mode().Perm() + if stat, ok := info.Sys().(*syscall.Stat_t); ok { + identity.uid = int(stat.Uid) + identity.gid = int(stat.Gid) + identity.owned = true + } + return identity +} +func applyIdentity(file *os.File, identity fileIdentity) error { + if identity.owned { + if err := file.Chown(identity.uid, identity.gid); err != nil { + return err + } + } + return file.Chmod(identity.mode) +} diff --git a/internal/deploy/ownership_other.go b/internal/deploy/ownership_other.go new file mode 100644 index 0000000..1e10094 --- /dev/null +++ b/internal/deploy/ownership_other.go @@ -0,0 +1,17 @@ +//go:build !linux + +// SPDX-License-Identifier: AGPL-3.0-only + +package deploy + +import "os" + +type fileIdentity struct{ mode os.FileMode } + +func identityFor(info os.FileInfo, fallback os.FileMode) fileIdentity { + if info != nil { + return fileIdentity{mode: info.Mode().Perm()} + } + return fileIdentity{mode: fallback} +} +func applyIdentity(file *os.File, identity fileIdentity) error { return file.Chmod(identity.mode) } diff --git a/internal/deploy/reconcile.go b/internal/deploy/reconcile.go new file mode 100644 index 0000000..b44d581 --- /dev/null +++ b/internal/deploy/reconcile.go @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package deploy + +import ( + "bytes" + "context" + "errors" + "os" + + "gamertan.com/tend/internal/config" + "gamertan.com/tend/internal/state" +) + +type Finding struct { + Code string `json:"code"` + Severity string `json:"severity"` + Message string `json:"message"` +} + +type ObservedReleaseIdentity struct { + Version string `json:"version"` + Commit string `json:"commit"` +} + +type ObservedState struct { + ActiveRelease string `json:"active_release,omitempty"` + PreviousRelease string `json:"previous_release,omitempty"` + ActiveIdentity *ObservedReleaseIdentity `json:"active_identity,omitempty"` + Units map[string]bool `json:"units"` + CandidateLease bool `json:"candidate_lease"` + CandidateUnit string `json:"candidate_unit,omitempty"` + CandidateUnitActive *bool `json:"candidate_unit_active,omitempty"` + LegacyCandidateUnit string `json:"legacy_candidate_unit,omitempty"` + LegacyCandidateActive *bool `json:"legacy_candidate_unit_active,omitempty"` + RouteHandlerMatches bool `json:"route_handler_matches"` + HandlerFileTarget string `json:"handler_file_target,omitempty"` +} + +type Reconciliation struct { + Service string `json:"service"` + Strategy string `json:"strategy"` + Mutation string `json:"mutation"` + Consistent bool `json:"consistent"` + StateInitialized bool `json:"state_initialized"` + State *state.Record `json:"state,omitempty"` + Observed ObservedState `json:"observed"` + Disposition string `json:"disposition"` + Findings []Finding `json:"findings"` +} + +// Reconcile observes configured units, release pointers, installed release +// identity, and the imported Caddy handler. It never acquires the deployment +// lock or mutates service state; proposed repairs remain an operator decision. +func (m Manager) Reconcile(ctx context.Context, cfg config.Config) (Reconciliation, error) { + if err := cfg.Validate(); err != nil { + return Reconciliation{}, err + } + if m.Operator == nil || m.ReadIdentity == nil { + return Reconciliation{}, errors.New("reconciliation dependencies are unavailable") + } + report := Reconciliation{ + Service: cfg.Service.Name, Strategy: cfg.Deployment.Strategy, Mutation: "none", + Observed: ObservedState{Units: map[string]bool{}}, Findings: []Finding{}, + } + add := func(code, severity, message string) { + report.Findings = append(report.Findings, Finding{Code: code, Severity: severity, Message: message}) + } + record, err := state.Load(cfg.Deployment.StateFile, cfg.Deployment.Root, cfg.Deployment.Strategy) + if err == nil { + report.State = &record + report.StateInitialized = true + } else if os.IsNotExist(err) { + add("state_uninitialized", "warning", "Tend has no validated state record for this service.") + } else { + add("state_invalid", "error", "The Tend state record could not be validated.") + } + + activeSlot := "" + activeUnit := "" + activeAddress := "" + handler := "" + template := "" + var candidateLease *state.CandidateLease + switch cfg.Deployment.Strategy { + case "singleton_candidate": + single := *cfg.Deployment.Singleton + activeSlot = "singleton" + activeUnit = single.Unit + activeAddress = single.Address + handler, template = single.CaddyHandler, single.CaddyHandlerTemplate + report.Observed.ActiveRelease = observeReleaseLink(cfg, single.CurrentLink, true, add) + report.Observed.PreviousRelease = observeReleaseLink(cfg, single.PreviousLink, false, add) + legacyCandidateUnit := cfg.Service.Name + "-tend-candidate.service" + candidateUnit := legacyCandidateUnit + lease, leaseErr := state.LoadCandidateLease(state.CandidateLeasePath(cfg.Deployment.StateFile), cfg.Deployment.Root, cfg.Service.Name) + if leaseErr == nil { + candidateLease = &lease + report.Observed.CandidateLease = true + candidateUnit = lease.Unit + } else if !os.IsNotExist(leaseErr) { + report.Observed.CandidateLease = true + candidateUnit = "" + add("candidate_lease_invalid", "error", "The operation-scoped candidate lease could not be validated.") + } else if report.State != nil && report.State.CandidateRelease != "" { + report.Observed.CandidateLease = true + candidateUnit = "" + add("legacy_candidate_lease", "error", "The state records a candidate release without an operation-scoped lease and requires manual review.") + } + if candidateUnit != "" { + report.Observed.CandidateUnit = candidateUnit + candidateActive, candidateErr := m.Operator.IsActive(ctx, candidateUnit) + if candidateErr != nil { + add("candidate_unit_unobservable", "error", "The transient candidate unit state could not be observed.") + } else { + report.Observed.CandidateUnitActive = &candidateActive + report.Observed.Units[candidateUnit] = candidateActive + } + } + if candidateUnit != legacyCandidateUnit { + report.Observed.LegacyCandidateUnit = legacyCandidateUnit + legacyActive, legacyErr := m.Operator.IsActive(ctx, legacyCandidateUnit) + if legacyErr != nil { + add("legacy_candidate_unit_unobservable", "error", "The legacy fixed candidate unit state could not be observed.") + } else { + report.Observed.LegacyCandidateActive = &legacyActive + report.Observed.Units[legacyCandidateUnit] = legacyActive + if legacyActive { + add("legacy_candidate_unit_active", "error", "A legacy fixed-name candidate remains active beside an operation-scoped lease.") + } + } + } + case "blue_green": + blueGreen := *cfg.Deployment.BlueGreen + handler, template = blueGreen.CaddyHandler, blueGreen.CaddyHandlerTemplate + activeSlot = blueGreen.BootstrapActive + if report.State != nil { + activeSlot = report.State.ActiveSlot + } + active := slotConfig(blueGreen, activeSlot) + previousName := "blue" + if activeSlot == "blue" { + previousName = "green" + } + previous := slotConfig(blueGreen, previousName) + activeUnit, activeAddress = active.Unit, active.Address + report.Observed.ActiveRelease = observeReleaseLink(cfg, active.Link, true, add) + report.Observed.PreviousRelease = observeReleaseLink(cfg, previous.Link, false, add) + for _, slot := range []config.Slot{blueGreen.Blue, blueGreen.Green} { + observeUnit(ctx, m.Operator, slot.Unit, report.Observed.Units, add) + } + } + if _, exists := report.Observed.Units[activeUnit]; !exists { + observeUnit(ctx, m.Operator, activeUnit, report.Observed.Units, add) + } + if active, observed := report.Observed.Units[activeUnit]; activeUnit != "" && observed && !active { + add("active_unit_inactive", "error", "The configured active service unit is not active.") + } + + if report.Observed.ActiveRelease != "" { + identity, identityErr := m.ReadIdentity(report.Observed.ActiveRelease) + if identityErr != nil { + add("active_identity_unreadable", "error", "The observed active release identity could not be validated.") + } else { + report.Observed.ActiveIdentity = &ObservedReleaseIdentity{Version: identity.Version, Commit: identity.Commit} + } + } + expectedHandler, renderErr := renderHandler(template, activeAddress) + actualHandler, readErr := os.ReadFile(handler) + if renderErr != nil || readErr != nil { + add("route_handler_unreadable", "error", "The configured Caddy handler or its template could not be validated.") + } else { + report.Observed.RouteHandlerMatches = bytes.Equal(expectedHandler, actualHandler) + if report.Observed.RouteHandlerMatches { + report.Observed.HandlerFileTarget = "installed" + } else if cfg.Deployment.Strategy == "singleton_candidate" { + candidateAddress := cfg.Deployment.Singleton.CandidateAddress + if candidateLease != nil { + candidateAddress = candidateLease.Address + } + candidateHandler, candidateErr := renderHandler(template, candidateAddress) + if candidateErr == nil && bytes.Equal(candidateHandler, actualHandler) { + report.Observed.HandlerFileTarget = "candidate" + } else { + report.Observed.HandlerFileTarget = "other" + } + } + if !report.Observed.RouteHandlerMatches { + add("route_handler_drift", "error", "The installed Caddy handler does not match the configured active upstream.") + } + } + + if report.State != nil { + if report.State.ActiveSlot != activeSlot || report.State.ActiveRelease != report.Observed.ActiveRelease { + add("active_release_drift", "error", "Recorded active state does not match the observed active release pointer.") + } + if report.State.PreviousRelease != report.Observed.PreviousRelease { + add("previous_release_drift", "warning", "Recorded rollback state does not match the observed previous release pointer.") + } + if cfg.Deployment.Strategy == "singleton_candidate" { + leased := report.Observed.CandidateLease && candidateLease != nil + candidateActive := report.Observed.CandidateUnitActive != nil && *report.Observed.CandidateUnitActive + if candidateLease != nil && report.State.CandidateRelease == "" { + add("candidate_lease_without_running_attempt", "error", "An operation-scoped candidate lease remains after the recorded attempt settled.") + } + if candidateLease != nil && report.State.CandidateRelease != "" && candidateLease.Release != report.State.CandidateRelease { + add("candidate_lease_release_mismatch", "error", "The operation-scoped candidate lease does not match the recorded candidate release.") + } + if leased && report.Observed.CandidateUnitActive != nil && !candidateActive { + add("inactive_candidate_lease", "error", "State retains a candidate lease but its operation-scoped unit is inactive.") + } + if !report.Observed.CandidateLease && candidateActive { + add("unleased_candidate_active", "error", "A transient candidate unit is active without a matching running attempt.") + } + if leased && candidateActive && report.Observed.HandlerFileTarget == "candidate" { + add("retained_candidate_routed", "warning", "The retained candidate appears in the handler file; do not stop it before establishing another healthy route.") + } + if leased && candidateActive && report.Observed.HandlerFileTarget != "candidate" { + add("candidate_active_not_routed", "warning", "The leased candidate is active but the handler file does not target it; cleanup remains an explicit reviewed operation.") + } + } + } + switch { + case !report.StateInitialized: + report.Disposition = "state_uninitialized" + case report.Observed.CandidateLease && report.Observed.CandidateUnit == "": + report.Disposition = "legacy_or_invalid_candidate_lease" + case report.Observed.CandidateLease && report.Observed.CandidateUnitActive != nil && *report.Observed.CandidateUnitActive && report.Observed.HandlerFileTarget == "candidate": + report.Disposition = "retained_candidate_handler_file" + case report.Observed.CandidateLease && report.Observed.CandidateUnitActive != nil && *report.Observed.CandidateUnitActive: + report.Disposition = "candidate_active_not_in_handler_file" + case report.Observed.CandidateLease: + report.Disposition = "inactive_candidate_lease" + case len(report.Findings) == 0: + report.Disposition = "settled" + default: + report.Disposition = "manual_review_required" + } + report.Consistent = len(report.Findings) == 0 + return report, nil +} + +func observeReleaseLink(cfg config.Config, link string, required bool, add func(string, string, string)) string { + release, err := resolveReleaseLink(cfg.Deployment.Root, link) + if err == nil { + return release + } + if !required && os.IsNotExist(err) { + return "" + } + code := "previous_pointer_unreadable" + message := "The configured previous release pointer could not be validated." + severity := "warning" + if required { + code = "active_pointer_unreadable" + message = "The configured active release pointer could not be validated." + severity = "error" + } + add(code, severity, message) + return "" +} + +func observeUnit(ctx context.Context, operator Operator, unit string, units map[string]bool, add func(string, string, string)) { + active, err := operator.IsActive(ctx, unit) + if err != nil { + add("unit_unobservable", "error", "A configured service unit state could not be observed.") + return + } + units[unit] = active +} diff --git a/internal/deploy/reconcile_test.go b/internal/deploy/reconcile_test.go new file mode 100644 index 0000000..8365382 --- /dev/null +++ b/internal/deploy/reconcile_test.go @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package deploy + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "gamertan.com/tend/internal/config" + "gamertan.com/tend/internal/state" +) + +func writeReleaseIdentity(t *testing.T, release, version, commit string) { + t.Helper() + body := `{"version":"` + version + `","commit":"` + commit + `"}` + if err := os.WriteFile(filepath.Join(release, "RELEASE.json"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + +func storeSingletonState(t *testing.T, cfg config.Config, active, previous string) { + t.Helper() + record := state.Record{ + SchemaVersion: state.SchemaVersion, + Strategy: "singleton_candidate", + DesiredRelease: active, + ActiveSlot: "singleton", + ActiveRelease: active, + PreviousRelease: previous, + LastAttemptRelease: active, + LastAttemptOutcome: "succeeded", + LastAttemptAt: time.Unix(100, 0).UTC().Format(time.RFC3339), + UpdatedAt: time.Unix(100, 0).UTC().Format(time.RFC3339), + } + if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil { + t.Fatal(err) + } +} + +func reconciliationManager(operator Operator) Manager { + return Manager{Operator: operator, ReadIdentity: readReleaseIdentity} +} + +func TestReconcileReportsHealthySingletonWithoutMutation(t *testing.T) { + cfg, active, previous := baseConfig(t, "singleton_candidate") + cfg.Deployment.Singleton, _ = singletonSettings(t, cfg, active) + if err := replaceSymlink(cfg.Deployment.Singleton.PreviousLink, previous); err != nil { + t.Fatal(err) + } + commit := strings.Repeat("a", 40) + writeReleaseIdentity(t, active, "v0.2.0-preview.2", commit) + storeSingletonState(t, cfg, active, previous) + operator := &fakeOperator{active: map[string]bool{ + cfg.Deployment.Singleton.Unit: true, + cfg.Service.Name + "-tend-candidate.service": false, + }} + + report, err := reconciliationManager(operator).Reconcile(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + if !report.Consistent || report.Mutation != "none" || !report.StateInitialized || len(report.Findings) != 0 { + t.Fatalf("report=%+v", report) + } + if report.Observed.ActiveRelease != active || report.Observed.PreviousRelease != previous || !report.Observed.RouteHandlerMatches { + t.Fatalf("observed=%+v", report.Observed) + } + if report.Observed.ActiveIdentity == nil || report.Observed.ActiveIdentity.Version != "v0.2.0-preview.2" || report.Observed.ActiveIdentity.Commit != commit { + t.Fatalf("identity=%+v", report.Observed.ActiveIdentity) + } + if !report.Observed.Units[cfg.Deployment.Singleton.Unit] || report.Observed.CandidateUnitActive == nil || *report.Observed.CandidateUnitActive { + t.Fatalf("units=%#v candidate=%v", report.Observed.Units, report.Observed.CandidateUnitActive) + } +} + +func TestReconcileReportsHealthyBlueGreenDeployment(t *testing.T) { + cfg, active, previous := baseConfig(t, "blue_green") + handler := filepath.Join(cfg.Deployment.Root, "handler.caddy") + template := filepath.Join(cfg.Deployment.Root, "handler.template") + if err := os.WriteFile(handler, []byte("reverse_proxy 127.0.0.1:8090\n"), 0o640); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(template, []byte("reverse_proxy {{UPSTREAM}}\n"), 0o644); err != nil { + t.Fatal(err) + } + blue := filepath.Join(cfg.Deployment.Root, "slots", "blue") + green := filepath.Join(cfg.Deployment.Root, "slots", "green") + if err := replaceSymlink(blue, active); err != nil { + t.Fatal(err) + } + if err := replaceSymlink(green, previous); err != nil { + t.Fatal(err) + } + cfg.Deployment.BlueGreen = &config.BlueGreen{ + CaddyConfig: filepath.Join(cfg.Deployment.Root, "Caddyfile"), CaddyHandler: handler, + CaddyHandlerTemplate: template, BootstrapActive: "blue", + Blue: config.Slot{Unit: "example-blue.service", Address: "127.0.0.1:8090", Link: blue}, + Green: config.Slot{Unit: "example-green.service", Address: "127.0.0.1:8091", Link: green}, + } + writeReleaseIdentity(t, active, "v0.2.0-preview.2", strings.Repeat("c", 40)) + record := state.Record{ + SchemaVersion: state.SchemaVersion, Strategy: "blue_green", DesiredRelease: active, + ActiveSlot: "blue", ActiveRelease: active, PreviousSlot: "green", PreviousRelease: previous, + LastAttemptRelease: active, LastAttemptOutcome: "succeeded", + LastAttemptAt: time.Unix(100, 0).UTC().Format(time.RFC3339), UpdatedAt: time.Unix(100, 0).UTC().Format(time.RFC3339), + } + if err := state.Store(cfg.Deployment.StateFile, cfg.Deployment.Root, record); err != nil { + t.Fatal(err) + } + operator := &fakeOperator{active: map[string]bool{ + "example-blue.service": true, "example-green.service": true, + }} + + report, err := reconciliationManager(operator).Reconcile(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + if !report.Consistent || report.Mutation != "none" || report.Observed.ActiveRelease != active || report.Observed.PreviousRelease != previous || !report.Observed.RouteHandlerMatches { + t.Fatalf("report=%+v", report) + } + if report.Observed.CandidateUnit != "" || report.Observed.CandidateUnitActive != nil { + t.Fatalf("unexpected candidate observation=%+v", report.Observed) + } +} + +func TestReconcileExplainsDriftWithoutRepairingIt(t *testing.T) { + cfg, recorded, observed := baseConfig(t, "singleton_candidate") + var handler string + cfg.Deployment.Singleton, handler = singletonSettings(t, cfg, recorded) + if err := replaceSymlink(cfg.Deployment.Singleton.PreviousLink, observed); err != nil { + t.Fatal(err) + } + storeSingletonState(t, cfg, recorded, observed) + if err := replaceSymlink(cfg.Deployment.Singleton.CurrentLink, observed); err != nil { + t.Fatal(err) + } + writeReleaseIdentity(t, observed, "v0.2.0-preview.3", strings.Repeat("b", 40)) + candidateHandler, err := renderHandler(cfg.Deployment.Singleton.CaddyHandlerTemplate, cfg.Deployment.Singleton.CandidateAddress) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(handler, candidateHandler, 0o640); err != nil { + t.Fatal(err) + } + beforeHandler, err := os.ReadFile(handler) + if err != nil { + t.Fatal(err) + } + operator := &fakeOperator{active: map[string]bool{ + cfg.Deployment.Singleton.Unit: false, + cfg.Service.Name + "-tend-candidate.service": true, + }} + + report, err := reconciliationManager(operator).Reconcile(context.Background(), cfg) + if err != nil { + t.Fatal(err) + } + if report.Consistent || report.Mutation != "none" { + t.Fatalf("report=%+v", report) + } + codes := map[string]bool{} + for _, finding := range report.Findings { + codes[finding.Code] = true + } + for _, code := range []string{"active_release_drift", "active_unit_inactive", "route_handler_drift", "unleased_candidate_active"} { + if !codes[code] { + t.Fatalf("missing %s in %#v", code, report.Findings) + } + } + target, err := resolveReleaseLink(cfg.Deployment.Root, cfg.Deployment.Singleton.CurrentLink) + if err != nil || target != observed { + t.Fatalf("current=%q err=%v", target, err) + } + afterHandler, err := os.ReadFile(handler) + if err != nil || string(afterHandler) != string(beforeHandler) { + t.Fatalf("handler changed err=%v", err) + } +} diff --git a/internal/deploy/release.go b/internal/deploy/release.go new file mode 100644 index 0000000..5222bd3 --- /dev/null +++ b/internal/deploy/release.go @@ -0,0 +1,318 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package deploy + +import ( + "archive/tar" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "gamertan.com/tend/internal/config" + "gamertan.com/tend/internal/packager" +) + +const maxArtifactSize int64 = 512 << 20 + +func prepareRelease(cfg config.Config, artifact, expected, approved string) (string, error) { + if err := checkArtifact(artifact, expected, approved); err != nil { + return "", err + } + if err := ensureTree(cfg.Deployment.Root); err != nil { + return "", err + } + releases := filepath.Join(cfg.Deployment.Root, "releases") + release := filepath.Join(releases, "sha256-"+expected) + if info, err := os.Lstat(release); err == nil { + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return "", errors.New("existing release is not a directory") + } + if err := validateRelease(cfg, release); err != nil { + return "", err + } + return release, nil + } else if !os.IsNotExist(err) { + return "", err + } + stage := filepath.Join(releases, ".tend-stage-"+expected) + if err := os.Mkdir(stage, 0o700); err != nil { + return "", err + } + ok := false + defer func() { + if !ok { + _ = os.RemoveAll(stage) + } + }() + if err := extractArtifact(artifact, stage); err != nil { + return "", err + } + if err := validateRelease(cfg, stage); err != nil { + return "", err + } + if err := os.Chmod(stage, 0o755); err != nil { + return "", err + } + if err := os.Rename(stage, release); err != nil { + return "", err + } + ok = true + return release, nil +} + +func inspectArtifact(cfg config.Config, artifact, expected, approved string) error { + if err := checkArtifact(artifact, expected, approved); err != nil { + return err + } + stage, err := os.MkdirTemp("", "tend-inspect-") + if err != nil { + return err + } + defer os.RemoveAll(stage) + if err := extractArtifact(artifact, stage); err != nil { + return err + } + return validateRelease(cfg, stage) +} +func checkArtifact(artifact, expected, approved string) error { + if expected != approved || len(expected) != 64 { + return errors.New("artifact digest was not explicitly approved") + } + if _, err := hex.DecodeString(expected); err != nil { + return errors.New("artifact digest is not hexadecimal") + } + if !filepath.IsAbs(artifact) || filepath.Clean(artifact) != artifact { + return errors.New("artifact path must be a clean absolute path") + } + info, err := os.Lstat(artifact) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Size() > maxArtifactSize { + return errors.New("artifact must be a bounded regular file") + } + actual, err := fileSHA(artifact) + if err != nil { + return err + } + if actual != expected { + return errors.New("artifact digest does not match") + } + return nil +} + +func ensureTree(root string) error { + if os.Geteuid() != 0 && strings.HasPrefix(root, "/opt/") { + return errors.New("deployment under /opt requires root") + } + if err := rejectSymlinkAncestors(root); err != nil { + return err + } + if err := os.MkdirAll(filepath.Join(root, "releases"), 0o755); err != nil { + return err + } + return rejectSymlinkAncestors(filepath.Join(root, "releases")) +} +func rejectSymlinkAncestors(path string) error { + clean := filepath.Clean(path) + parts := strings.Split(strings.TrimPrefix(clean, string(filepath.Separator)), string(filepath.Separator)) + current := string(filepath.Separator) + for _, part := range parts { + current = filepath.Join(current, part) + info, err := os.Lstat(current) + if os.IsNotExist(err) { + continue + } + if err != nil { + return err + } + if info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("symlink ancestor refused: %s", current) + } + if !info.IsDir() { + return fmt.Errorf("non-directory ancestor refused: %s", current) + } + } + return nil +} + +func extractArtifact(artifact, stage string) error { + file, err := os.Open(artifact) + if err != nil { + return err + } + defer file.Close() + gz, err := gzip.NewReader(file) + if err != nil { + return err + } + defer gz.Close() + tr := tar.NewReader(io.LimitReader(gz, maxArtifactSize)) + files := 0 + var total int64 + for { + header, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + clean := filepath.Clean(filepath.FromSlash(header.Name)) + parts := strings.Split(clean, string(filepath.Separator)) + if len(parts) == 1 && header.Typeflag == tar.TypeDir { + continue + } + if len(parts) != 2 || parts[0] != "bundle" || parts[1] == "" || parts[1] == "." || parts[1] == ".." { + return fmt.Errorf("unsafe archive path %q", header.Name) + } + if header.Typeflag != tar.TypeReg || header.Size < 0 { + return errors.New("archive may contain only regular files") + } + files++ + total += header.Size + if files > 16 || total > maxArtifactSize { + return errors.New("artifact exceeds extraction bounds") + } + target := filepath.Join(stage, parts[1]) + mode := os.FileMode(0o644) + if !strings.HasSuffix(parts[1], ".json") && parts[1] != "SHA256SUMS" { + mode = 0o755 + } + out, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, mode) + if err != nil { + return err + } + if _, err := io.CopyN(out, tr, header.Size); err != nil { + _ = out.Close() + return err + } + // OpenFile modes are filtered through the caller's umask. Tend is + // commonly invoked by a root account with umask 0077, while release + // binaries must remain executable by their dedicated service users. + // Reapply the validated, name-derived mode explicitly before the file + // becomes part of an immutable release. + if err := out.Chmod(mode); err != nil { + _ = out.Close() + return err + } + if err := out.Sync(); err != nil { + _ = out.Close() + return err + } + if err := out.Close(); err != nil { + return err + } + } + if files < 5 { + return errors.New("artifact is incomplete") + } + return nil +} + +func validateRelease(cfg config.Config, release string) error { + manifestPath := filepath.Join(release, "RELEASE.json") + b, err := os.ReadFile(manifestPath) + if err != nil { + return err + } + dec := json.NewDecoder(strings.NewReader(string(b))) + dec.DisallowUnknownFields() + var manifest packager.Manifest + if err := dec.Decode(&manifest); err != nil { + return err + } + var trailing any + if err := dec.Decode(&trailing); !errors.Is(err, io.EOF) { + return errors.New("release manifest contains trailing data") + } + if manifest.SchemaVersion != 1 || manifest.Service != cfg.Service.Name || manifest.Binary != cfg.Build.Binary || manifest.GOOS != "linux" || manifest.GOARCH != "amd64" || manifest.CGOEnabled { + return errors.New("release manifest does not match configuration") + } + if matched, _ := regexp.MatchString(`^[0-9a-f]{40}$`, manifest.Commit); !matched { + return errors.New("release manifest commit is invalid") + } + binary := filepath.Join(release, cfg.Build.Binary) + sum, err := fileSHA(binary) + if err != nil { + return err + } + if sum != manifest.BinarySHA256 { + return errors.New("release binary digest does not match manifest") + } + if err := verifySums(release); err != nil { + return err + } + entries, err := os.ReadDir(release) + if err != nil { + return err + } + allowed := map[string]bool{cfg.Build.Binary: true, "BUILDINFO.json": true, "RELEASE.json": true, "SBOM.spdx.json": true, "SHA256SUMS": true} + if len(entries) != len(allowed) { + return errors.New("release contains unexpected files") + } + for _, entry := range entries { + if !allowed[entry.Name()] || !entry.Type().IsRegular() { + return fmt.Errorf("unexpected release entry %s", entry.Name()) + } + } + return nil +} +func verifySums(release string) error { + b, err := os.ReadFile(filepath.Join(release, "SHA256SUMS")) + if err != nil { + return err + } + lines := strings.Split(strings.TrimSpace(string(b)), "\n") + if len(lines) != 4 { + return errors.New("SHA256SUMS must cover four release files") + } + seen := map[string]bool{} + for _, line := range lines { + fields := strings.Fields(line) + if len(fields) != 2 || len(fields[0]) != 64 { + return errors.New("malformed SHA256SUMS") + } + name := fields[1] + if filepath.Base(name) != name || seen[name] { + return errors.New("unsafe or duplicate checksum entry") + } + seen[name] = true + actual, err := fileSHA(filepath.Join(release, name)) + if err != nil { + return err + } + if actual != fields[0] { + return fmt.Errorf("checksum mismatch for %s", name) + } + } + required := []string{"BUILDINFO.json", "RELEASE.json", "SBOM.spdx.json"} + sort.Strings(required) + for _, name := range required { + if !seen[name] { + return fmt.Errorf("checksum omitted %s", name) + } + } + return nil +} +func fileSHA(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/deploy/release_mode_linux_test.go b/internal/deploy/release_mode_linux_test.go new file mode 100644 index 0000000..0ce20af --- /dev/null +++ b/internal/deploy/release_mode_linux_test.go @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//go:build linux + +package deploy + +import ( + "archive/tar" + "compress/gzip" + "os" + "path/filepath" + "syscall" + "testing" +) + +func TestExtractArtifactAppliesReleaseModesUnderRestrictiveUmask(t *testing.T) { + dir := t.TempDir() + artifact := filepath.Join(dir, "release.tar.gz") + file, err := os.OpenFile(artifact, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + gz := gzip.NewWriter(file) + tw := tar.NewWriter(gz) + entries := map[string][]byte{ + "bundle/app": []byte("executable"), + "bundle/BUILDINFO.json": []byte("{}"), + "bundle/RELEASE.json": []byte("{}"), + "bundle/SBOM.spdx.json": []byte("{}"), + "bundle/SHA256SUMS": []byte("checksums"), + } + for name, body := range entries { + if err := tw.WriteHeader(&tar.Header{Name: name, Typeflag: tar.TypeReg, Mode: 0o600, Size: int64(len(body))}); err != nil { + t.Fatal(err) + } + if _, err := tw.Write(body); err != nil { + t.Fatal(err) + } + } + if err := tw.Close(); err != nil { + t.Fatal(err) + } + if err := gz.Close(); err != nil { + t.Fatal(err) + } + if err := file.Close(); err != nil { + t.Fatal(err) + } + + oldUmask := syscall.Umask(0o077) + t.Cleanup(func() { syscall.Umask(oldUmask) }) + stage := filepath.Join(dir, "stage") + if err := os.Mkdir(stage, 0o700); err != nil { + t.Fatal(err) + } + if err := extractArtifact(artifact, stage); err != nil { + t.Fatal(err) + } + + for name, want := range map[string]os.FileMode{ + "app": 0o755, + "BUILDINFO.json": 0o644, + "RELEASE.json": 0o644, + "SBOM.spdx.json": 0o644, + "SHA256SUMS": 0o644, + } { + info, err := os.Stat(filepath.Join(stage, name)) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != want { + t.Fatalf("%s mode=%#o want=%#o", name, got, want) + } + } +} diff --git a/internal/deploy/release_test.go b/internal/deploy/release_test.go new file mode 100644 index 0000000..fe5e059 --- /dev/null +++ b/internal/deploy/release_test.go @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package deploy + +import ( + "archive/tar" + "compress/gzip" + "os" + "path/filepath" + "testing" +) + +func writeHostileArchive(t *testing.T, name string, typeflag byte) { + t.Helper() + path := filepath.Join(t.TempDir(), "bad.tar.gz") + file, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + gz := gzip.NewWriter(file) + tw := tar.NewWriter(gz) + body := []byte("x") + if err := tw.WriteHeader(&tar.Header{Name: name, Typeflag: typeflag, Mode: 0o644, Size: int64(len(body))}); err != nil { + t.Fatal(err) + } + if typeflag == tar.TypeReg { + _, _ = tw.Write(body) + } + _ = tw.Close() + _ = gz.Close() + _ = file.Close() + stage := filepath.Join(t.TempDir(), "stage") + _ = os.Mkdir(stage, 0o700) + if err := extractArtifact(path, stage); err == nil { + t.Fatalf("accepted hostile entry %q type %d", name, typeflag) + } +} +func TestExtractionRejectsTraversalAndLinks(t *testing.T) { + writeHostileArchive(t, "bundle/../../escape", tar.TypeReg) + writeHostileArchive(t, "bundle/link", tar.TypeSymlink) +} diff --git a/internal/eventlog/eventlog.go b/internal/eventlog/eventlog.go new file mode 100644 index 0000000..5146dfe --- /dev/null +++ b/internal/eventlog/eventlog.go @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package eventlog + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "syscall" + "time" +) + +const Version = 1 + +var safeValue = regexp.MustCompile(`^[A-Za-z0-9._:/@+-]{1,256}$`) +var hexDigest = regexp.MustCompile(`^[0-9a-f]{64}$`) +var gitCommit = regexp.MustCompile(`^[0-9a-f]{40}$`) +var operationID = regexp.MustCompile(`^[0-9a-f]{32}$`) + +type Event struct { + Version int `json:"version"` + OperationID string `json:"operation_id"` + Service string `json:"service"` + ArtifactDigest string `json:"artifact_digest"` + Commit string `json:"commit"` + ReleaseVersion string `json:"release_version"` + Phase string `json:"phase"` + Slot string `json:"slot,omitempty"` + DurationMillis int64 `json:"duration_ms"` + Outcome string `json:"outcome"` + ObservedAt string `json:"observed_at"` +} + +func OperationID() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", errors.New("cryptographic randomness unavailable") + } + return hex.EncodeToString(b), nil +} + +func Append(path string, event Event) error { + if err := event.validate(); err != nil { + return err + } + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return errors.New("event log path must be absolute and clean") + } + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return fmt.Errorf("create event directory: %w", err) + } + if info, err := os.Lstat(path); err == nil { + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o022 != 0 { + return errors.New("event log must be a non-writable regular non-symlink file") + } + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("inspect event log: %w", err) + } + b, err := json.Marshal(event) + if err != nil { + return fmt.Errorf("encode deployment event: %w", err) + } + if len(b) > 4096 { + return errors.New("deployment event exceeds bound") + } + b = append(b, '\n') + fd, err := syscall.Open(path, syscall.O_WRONLY|syscall.O_APPEND|syscall.O_CREAT|syscall.O_CLOEXEC|syscall.O_NOFOLLOW, 0o640) + if err != nil { + return fmt.Errorf("open event log: %w", err) + } + file := os.NewFile(uintptr(fd), path) + if file == nil { + _ = syscall.Close(fd) + return errors.New("open event log file") + } + defer file.Close() + n, err := file.Write(b) + if err != nil || n != len(b) { + return errors.New("write complete deployment event") + } + if err := file.Sync(); err != nil { + return fmt.Errorf("sync deployment event: %w", err) + } + return nil +} + +func (e Event) validate() error { + if e.Version != Version || !operationID.MatchString(e.OperationID) { + return errors.New("deployment event identity is invalid") + } + if !hexDigest.MatchString(e.ArtifactDigest) || !gitCommit.MatchString(e.Commit) { + return errors.New("deployment event provenance is invalid") + } + for label, value := range map[string]string{"service": e.Service, "artifact_digest": e.ArtifactDigest, "commit": e.Commit, "release_version": e.ReleaseVersion, "phase": e.Phase, "outcome": e.Outcome} { + if !safeValue.MatchString(value) || strings.ContainsRune(value, '\x00') { + return fmt.Errorf("deployment event %s is invalid", label) + } + } + if e.Slot != "" && !safeValue.MatchString(e.Slot) { + return errors.New("deployment event slot is invalid") + } + if e.DurationMillis < 0 { + return errors.New("deployment event duration is invalid") + } + if _, err := time.Parse(time.RFC3339Nano, e.ObservedAt); err != nil { + return errors.New("deployment event timestamp is invalid") + } + return nil +} diff --git a/internal/eventlog/eventlog_test.go b/internal/eventlog/eventlog_test.go new file mode 100644 index 0000000..1351c6f --- /dev/null +++ b/internal/eventlog/eventlog_test.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package eventlog + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestAppendBoundedEvent(t *testing.T) { + path := filepath.Join(t.TempDir(), "events.jsonl") + id, err := OperationID() + if err != nil { + t.Fatal(err) + } + event := Event{Version: 1, OperationID: id, Service: "site", ArtifactDigest: strings.Repeat("a", 64), Commit: strings.Repeat("b", 40), ReleaseVersion: "v0.2.0-preview.1", Phase: "activation", Slot: "green", Outcome: "succeeded", ObservedAt: time.Now().UTC().Format(time.RFC3339Nano)} + if err := Append(path, event); err != nil { + t.Fatal(err) + } + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var decoded Event + if err := json.Unmarshal(b, &decoded); err != nil { + t.Fatal(err) + } + if decoded.OperationID != id || strings.Contains(string(b), "secret") { + t.Fatalf("unexpected event: %s", b) + } + if info, _ := os.Stat(path); info.Mode().Perm() != 0o640 { + t.Fatalf("mode=%04o", info.Mode().Perm()) + } +} + +func TestAppendRejectsUnboundedValuesAndSymlink(t *testing.T) { + id, _ := OperationID() + event := Event{Version: 1, OperationID: id, Service: "site\nsecret", ArtifactDigest: "digest", Commit: "commit", ReleaseVersion: "version", Phase: "activation", Outcome: "failed", ObservedAt: time.Now().UTC().Format(time.RFC3339Nano)} + if err := Append(filepath.Join(t.TempDir(), "events.jsonl"), event); err == nil { + t.Fatal("expected unsafe value rejection") + } + dir := t.TempDir() + target := filepath.Join(dir, "target") + if err := os.WriteFile(target, nil, 0o640); err != nil { + t.Fatal(err) + } + link := filepath.Join(dir, "events.jsonl") + if err := os.Symlink(target, link); err != nil { + t.Skip(err) + } + event.Service = "site" + if err := Append(link, event); err == nil { + t.Fatal("expected symlink rejection") + } +} diff --git a/internal/packager/packager.go b/internal/packager/packager.go new file mode 100644 index 0000000..3fa3959 --- /dev/null +++ b/internal/packager/packager.go @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package packager + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/sha256" + "debug/buildinfo" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" + + "gamertan.com/tend/internal/config" + "gamertan.com/tend/internal/process" + "gamertan.com/tend/internal/provenance" +) + +var versionPattern = regexp.MustCompile(`^v[0-9]+\.[0-9]+\.[0-9]+-preview\.[1-9][0-9]*$`) + +type Result struct { + Artifact string `json:"artifact"` + SHA256 string `json:"sha256"` + Commit string `json:"commit"` + Version string `json:"version"` +} +type Manifest struct { + SchemaVersion int `json:"schema_version"` + Service string `json:"service"` + Version string `json:"version"` + Commit string `json:"commit"` + SourceEpoch int64 `json:"source_date_epoch"` + GOOS string `json:"goos"` + GOARCH string `json:"goarch"` + CGOEnabled bool `json:"cgo_enabled"` + Binary string `json:"binary"` + BinarySHA256 string `json:"binary_sha256"` + GoVersion string `json:"go_version"` + ModulePath string `json:"module_path"` + ModuleVersion string `json:"module_version"` +} +type buildRecord struct { + GoVersion string `json:"go_version"` + Path string `json:"path"` + Main moduleRecord `json:"main"` + Deps []moduleRecord `json:"dependencies"` + Settings []settingRecord `json:"settings"` +} +type moduleRecord struct { + Path string `json:"path"` + Version string `json:"version"` + Sum string `json:"sum,omitempty"` +} +type settingRecord struct { + Key string `json:"key"` + Value string `json:"value"` +} +type spdxDocument struct { + SPDXVersion string `json:"spdxVersion"` + DataLicense string `json:"dataLicense"` + SPDXID string `json:"SPDXID"` + Name string `json:"name"` + DocumentNamespace string `json:"documentNamespace"` + CreationInfo spdxCreation `json:"creationInfo"` + Packages []spdxPackage `json:"packages"` + Relationships []spdxRelationship `json:"relationships"` +} +type spdxCreation struct { + Created string `json:"created"` + Creators []string `json:"creators"` +} +type spdxPackage struct { + Name string `json:"name"` + SPDXID string `json:"SPDXID"` + VersionInfo string `json:"versionInfo"` + DownloadLocation string `json:"downloadLocation"` + FilesAnalyzed bool `json:"filesAnalyzed"` + LicenseConcluded string `json:"licenseConcluded"` + LicenseDeclared string `json:"licenseDeclared"` +} +type spdxRelationship struct { + SPDXElementID string `json:"spdxElementId"` + RelationshipType string `json:"relationshipType"` + RelatedSPDXElement string `json:"relatedSpdxElement"` +} + +func Package(ctx context.Context, runner process.Runner, cfg config.Config, sourceDir, outDir, version string) (Result, error) { + if !versionPattern.MatchString(version) { + return Result{}, errors.New("version must use vX.Y.Z-preview.N") + } + if !filepath.IsAbs(sourceDir) || !filepath.IsAbs(outDir) { + return Result{}, errors.New("source and output directories must be absolute") + } + source, err := provenance.Inspect(ctx, runner, sourceDir, cfg.Build.Branch) + if err != nil { + return Result{}, err + } + if err := verifyModules(ctx, runner, sourceDir); err != nil { + return Result{}, err + } + work, err := os.MkdirTemp("", "tend-package-") + if err != nil { + return Result{}, err + } + defer os.RemoveAll(work) + first := filepath.Join(work, "first", cfg.Build.Binary) + second := filepath.Join(work, "second", cfg.Build.Binary) + if err := build(ctx, runner, cfg, source, sourceDir, version, first); err != nil { + return Result{}, err + } + if err := build(ctx, runner, cfg, source, sourceDir, version, second); err != nil { + return Result{}, err + } + firstSHA, err := fileSHA(first) + if err != nil { + return Result{}, err + } + secondSHA, err := fileSHA(second) + if err != nil { + return Result{}, err + } + if firstSHA != secondSHA { + return Result{}, errors.New("two clean builds were not byte-identical") + } + record, err := readBuildRecord(first) + if err != nil { + return Result{}, err + } + if record.Main.Path == "" { + return Result{}, errors.New("built binary has no main module provenance") + } + if err := verifyBuildSettings(record, source.Commit); err != nil { + return Result{}, err + } + manifest := Manifest{SchemaVersion: 1, Service: cfg.Service.Name, Version: version, Commit: source.Commit, SourceEpoch: source.Epoch, GOOS: "linux", GOARCH: "amd64", CGOEnabled: false, Binary: cfg.Build.Binary, BinarySHA256: firstSHA, GoVersion: record.GoVersion, ModulePath: record.Main.Path, ModuleVersion: record.Main.Version} + bundle := filepath.Join(work, "bundle") + if err := os.Mkdir(bundle, 0o700); err != nil { + return Result{}, err + } + if err := copyFile(first, filepath.Join(bundle, cfg.Build.Binary), 0o755); err != nil { + return Result{}, err + } + if err := writeJSON(filepath.Join(bundle, "RELEASE.json"), manifest, 0o644); err != nil { + return Result{}, err + } + if err := writeJSON(filepath.Join(bundle, "BUILDINFO.json"), record, 0o644); err != nil { + return Result{}, err + } + if err := writeJSON(filepath.Join(bundle, "SBOM.spdx.json"), makeSPDX(cfg, version, source, record), 0o644); err != nil { + return Result{}, err + } + if err := writeSums(bundle, []string{cfg.Build.Binary, "BUILDINFO.json", "RELEASE.json", "SBOM.spdx.json"}); err != nil { + return Result{}, err + } + if err := os.MkdirAll(outDir, 0o755); err != nil { + return Result{}, err + } + artifactName := fmt.Sprintf("%s-%s-linux-amd64.tar.gz", cfg.Service.Name, strings.TrimPrefix(version, "v")) + artifact := filepath.Join(outDir, artifactName) + if err := writeArchive(artifact, bundle, source.Epoch); err != nil { + return Result{}, err + } + artifactSHA, err := fileSHA(artifact) + if err != nil { + return Result{}, err + } + if err := os.WriteFile(artifact+".sha256", []byte(artifactSHA+" "+artifactName+"\n"), 0o644); err != nil { + return Result{}, err + } + return Result{Artifact: artifact, SHA256: artifactSHA, Commit: source.Commit, Version: version}, nil +} + +func verifyModules(ctx context.Context, runner process.Runner, dir string) error { + out, err := runner.Run(ctx, dir, map[string]string{"GOWORK": "off", "GOFLAGS": "-mod=readonly"}, "go", "list", "-m", "-json", "all") + if err != nil { + return fmt.Errorf("list modules: %w", err) + } + dec := json.NewDecoder(strings.NewReader(string(out))) + count := 0 + for { + var module struct { + Path, Version string + Main bool + Replace *json.RawMessage + } + if err := dec.Decode(&module); errors.Is(err, io.EOF) { + break + } else if err != nil { + return fmt.Errorf("decode module graph: %w", err) + } + count++ + if module.Replace != nil { + return fmt.Errorf("module %s uses a replacement", module.Path) + } + if !module.Main && module.Version == "" { + return fmt.Errorf("module %s is not pinned", module.Path) + } + } + if count == 0 { + return errors.New("module graph is empty") + } + return nil +} + +func build(ctx context.Context, runner process.Runner, cfg config.Config, source provenance.Source, dir, version, output string) error { + if err := os.MkdirAll(filepath.Dir(output), 0o700); err != nil { + return err + } + ldflags := []string{"-s", "-w"} + date := time.Unix(source.Epoch, 0).UTC().Format(time.RFC3339) + pairs := [][2]string{{cfg.Build.VersionSymbol, version}, {cfg.Build.CommitSymbol, source.Commit}, {cfg.Build.DateSymbol, date}} + for _, pair := range pairs { + if pair[0] != "" { + ldflags = append(ldflags, "-X", pair[0]+"="+pair[1]) + } + } + args := []string{"build", "-mod=readonly", "-trimpath", "-buildvcs=true", "-ldflags", strings.Join(ldflags, " "), "-o", output, cfg.Build.Package} + env := map[string]string{"GOWORK": "off", "GOFLAGS": "-mod=readonly", "GOOS": "linux", "GOARCH": "amd64", "CGO_ENABLED": "0", "SOURCE_DATE_EPOCH": fmt.Sprint(source.Epoch)} + if _, err := runner.Run(ctx, dir, env, "go", args...); err != nil { + return fmt.Errorf("build candidate: %w", err) + } + return nil +} + +func readBuildRecord(path string) (buildRecord, error) { + info, err := buildinfo.ReadFile(path) + if err != nil { + return buildRecord{}, fmt.Errorf("read Go build info: %w", err) + } + record := buildRecord{GoVersion: info.GoVersion, Path: info.Path, Main: moduleRecord{Path: info.Main.Path, Version: info.Main.Version, Sum: info.Main.Sum}} + for _, dep := range info.Deps { + if dep.Replace != nil { + return buildRecord{}, fmt.Errorf("built binary contains replacement for %s", dep.Path) + } + record.Deps = append(record.Deps, moduleRecord{Path: dep.Path, Version: dep.Version, Sum: dep.Sum}) + } + for _, setting := range info.Settings { + record.Settings = append(record.Settings, settingRecord{Key: setting.Key, Value: setting.Value}) + } + sort.Slice(record.Deps, func(i, j int) bool { return record.Deps[i].Path < record.Deps[j].Path }) + sort.Slice(record.Settings, func(i, j int) bool { return record.Settings[i].Key < record.Settings[j].Key }) + return record, nil +} + +func verifyBuildSettings(record buildRecord, commit string) error { + settings := map[string]string{} + for _, setting := range record.Settings { + settings[setting.Key] = setting.Value + } + for key, expected := range map[string]string{"vcs.revision": commit, "vcs.modified": "false", "GOOS": "linux", "GOARCH": "amd64", "CGO_ENABLED": "0"} { + if settings[key] != expected { + return fmt.Errorf("build setting %s is %q, expected %q", key, settings[key], expected) + } + } + return nil +} +func makeSPDX(cfg config.Config, version string, source provenance.Source, record buildRecord) spdxDocument { + created := time.Unix(source.Epoch, 0).UTC().Format("2006-01-02T15:04:05Z") + modules := append([]moduleRecord{record.Main}, record.Deps...) + doc := spdxDocument{SPDXVersion: "SPDX-2.3", DataLicense: "CC0-1.0", SPDXID: "SPDXRef-DOCUMENT", Name: cfg.Service.Name + "-" + version, DocumentNamespace: "https://gamertan.com/tend/sbom/" + source.Commit + "/" + cfg.Service.Name, CreationInfo: spdxCreation{Created: created, Creators: []string{"Tool: gamertan.com/tend"}}} + for i, module := range modules { + id := fmt.Sprintf("SPDXRef-Package-%d", i+1) + versionInfo := module.Version + if versionInfo == "" { + versionInfo = source.Commit + } + doc.Packages = append(doc.Packages, spdxPackage{Name: module.Path, SPDXID: id, VersionInfo: versionInfo, DownloadLocation: "NOASSERTION", FilesAnalyzed: false, LicenseConcluded: "NOASSERTION", LicenseDeclared: "NOASSERTION"}) + doc.Relationships = append(doc.Relationships, spdxRelationship{SPDXElementID: "SPDXRef-DOCUMENT", RelationshipType: "DESCRIBES", RelatedSPDXElement: id}) + } + return doc +} +func writeJSON(path string, value any, mode os.FileMode) error { + b, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + b = append(b, '\n') + return os.WriteFile(path, b, mode) +} +func writeSums(dir string, names []string) error { + sort.Strings(names) + var b strings.Builder + for _, name := range names { + sum, err := fileSHA(filepath.Join(dir, name)) + if err != nil { + return err + } + fmt.Fprintf(&b, "%s %s\n", sum, name) + } + return os.WriteFile(filepath.Join(dir, "SHA256SUMS"), []byte(b.String()), 0o644) +} +func writeArchive(path, bundle string, epoch int64) error { + tmp := path + ".tmp" + _ = os.Remove(tmp) + file, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644) + if err != nil { + return err + } + ok := false + defer func() { + _ = file.Close() + if !ok { + _ = os.Remove(tmp) + } + }() + gz := gzip.NewWriter(file) + gz.Header.ModTime = time.Unix(0, 0) + gz.Header.OS = 255 + tw := tar.NewWriter(gz) + entries, err := os.ReadDir(bundle) + if err != nil { + return err + } + names := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.Type().IsRegular() { + names = append(names, entry.Name()) + } + } + sort.Strings(names) + root := filepath.Base(bundle) + if err := tw.WriteHeader(&tar.Header{Name: root + "/", Typeflag: tar.TypeDir, Mode: 0o755, ModTime: time.Unix(epoch, 0), Uid: 0, Gid: 0}); err != nil { + return err + } + for _, name := range names { + data, err := os.ReadFile(filepath.Join(bundle, name)) + if err != nil { + return err + } + mode := int64(0o644) + if name != "BUILDINFO.json" && !strings.HasSuffix(name, ".json") && name != "SHA256SUMS" { + mode = 0o755 + } + if err := tw.WriteHeader(&tar.Header{Name: root + "/" + name, Typeflag: tar.TypeReg, Mode: mode, Size: int64(len(data)), ModTime: time.Unix(epoch, 0), Uid: 0, Gid: 0}); err != nil { + return err + } + if _, err := tw.Write(data); err != nil { + return err + } + } + if err := tw.Close(); err != nil { + return err + } + if err := gz.Close(); err != nil { + return err + } + if err := file.Sync(); err != nil { + return err + } + if err := file.Close(); err != nil { + return err + } + if err := os.Rename(tmp, path); err != nil { + return err + } + ok = true + return nil +} +func copyFile(source, target string, mode os.FileMode) error { + in, err := os.Open(source) + if err != nil { + return err + } + defer in.Close() + out, err := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, mode) + if err != nil { + return err + } + if _, err := io.Copy(out, in); err != nil { + _ = out.Close() + return err + } + if err := out.Sync(); err != nil { + _ = out.Close() + return err + } + return out.Close() +} +func fileSHA(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/packager/packager_test.go b/internal/packager/packager_test.go new file mode 100644 index 0000000..ec9381e --- /dev/null +++ b/internal/packager/packager_test.go @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package packager + +import ( + "archive/tar" + "compress/gzip" + "errors" + "io" + "os" + "path/filepath" + "testing" +) + +func TestVersionPattern(t *testing.T) { + for _, v := range []string{"v0.1.0-preview.1", "v12.3.4-preview.99"} { + if !versionPattern.MatchString(v) { + t.Errorf("rejected %q", v) + } + } + for _, v := range []string{"v0.1.0", "0.1.0-preview.1", "v0.1.0-preview.0", "v0.1.0-preview.01", "v0.1.0-preview.1+dirty"} { + if versionPattern.MatchString(v) { + t.Errorf("accepted %q", v) + } + } +} +func TestArchiveHasOnlyRegularBundleEntries(t *testing.T) { + dir := t.TempDir() + bundle := filepath.Join(dir, "bundle") + if err := os.Mkdir(bundle, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(bundle, "app"), []byte("binary"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(bundle, "RELEASE.json"), []byte("{}\n"), 0o644); err != nil { + t.Fatal(err) + } + archive := filepath.Join(dir, "out.tar.gz") + if err := writeArchive(archive, bundle, 1); err != nil { + t.Fatal(err) + } + f, err := os.Open(archive) + if err != nil { + t.Fatal(err) + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + t.Fatal(err) + } + defer gz.Close() + tr := tar.NewReader(gz) + seen := 0 + for { + h, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatal(err) + } + if h.Name == "bundle/" { + continue + } + if h.Typeflag != tar.TypeReg { + t.Fatalf("unexpected type %d", h.Typeflag) + } + if filepath.IsAbs(h.Name) || filepath.Clean(h.Name) != h.Name { + t.Fatalf("unsafe path %q", h.Name) + } + seen++ + } + if seen != 2 { + t.Fatalf("saw %d files", seen) + } +} diff --git a/internal/process/run.go b/internal/process/run.go new file mode 100644 index 0000000..4162fce --- /dev/null +++ b/internal/process/run.go @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package process + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "sort" + "strings" +) + +const maxOutput = 4 << 20 + +type Runner interface { + Run(ctx context.Context, dir string, env map[string]string, name string, args ...string) ([]byte, error) +} + +type ExecRunner struct{} + +func (ExecRunner) Run(ctx context.Context, dir string, env map[string]string, name string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Dir = dir + cmd.Env = mergeEnv(os.Environ(), env) + var output limitedBuffer + cmd.Stdout = &output + cmd.Stderr = &output + err := cmd.Run() + if err != nil { + return output.Bytes(), fmt.Errorf("%s failed: %w: %s", name, err, strings.TrimSpace(output.String())) + } + return output.Bytes(), nil +} + +type limitedBuffer struct{ bytes.Buffer } + +func (b *limitedBuffer) Write(p []byte) (int, error) { + written := len(p) + remaining := maxOutput - b.Len() + if remaining > 0 { + if len(p) > remaining { + p = p[:remaining] + } + _, _ = b.Buffer.Write(p) + } + return written, nil +} + +func mergeEnv(base []string, extra map[string]string) []string { + values := make(map[string]string, len(base)+len(extra)) + for _, pair := range base { + key, value, ok := strings.Cut(pair, "=") + if ok { + values[key] = value + } + } + for key, value := range extra { + values[key] = value + } + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + out := make([]string, 0, len(keys)) + for _, key := range keys { + out = append(out, key+"="+values[key]) + } + return out +} diff --git a/internal/provenance/git.go b/internal/provenance/git.go new file mode 100644 index 0000000..bb1ddd2 --- /dev/null +++ b/internal/provenance/git.go @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package provenance + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "strconv" + "strings" + + "gamertan.com/tend/internal/process" +) + +type Source struct { + Commit string + Epoch int64 +} + +func Inspect(ctx context.Context, runner process.Runner, dir, branch string) (Source, error) { + gitDir, err := gitPath(ctx, runner, dir, "--git-dir") + if err != nil { + return Source{}, fmt.Errorf("inspect Git directory: %w", err) + } + commonDir, err := gitPath(ctx, runner, dir, "--git-common-dir") + if err != nil { + return Source{}, fmt.Errorf("inspect Git common directory: %w", err) + } + if gitDir != commonDir { + return Source{}, errors.New("release packaging does not yet support linked Git worktrees; use a clean standalone clone of the exact pushed commit") + } + status, err := runner.Run(ctx, dir, nil, "git", "status", "--porcelain=v1", "--untracked-files=all") + if err != nil { + return Source{}, err + } + if len(status) != 0 { + return Source{}, errors.New("source checkout is not clean") + } + commitOut, err := runner.Run(ctx, dir, nil, "git", "rev-parse", "HEAD") + if err != nil { + return Source{}, err + } + commit := strings.TrimSpace(string(commitOut)) + if len(commit) != 40 { + return Source{}, errors.New("source commit is not a full SHA-1 object id") + } + remoteOut, err := runner.Run(ctx, dir, nil, "git", "ls-remote", "--exit-code", "origin", "refs/heads/"+branch) + if err != nil { + return Source{}, fmt.Errorf("verify pushed commit: %w", err) + } + fields := strings.Fields(string(remoteOut)) + if len(fields) != 2 || fields[0] != commit || fields[1] != "refs/heads/"+branch { + return Source{}, errors.New("HEAD is not the exact pushed branch commit") + } + epochOut, err := runner.Run(ctx, dir, nil, "git", "show", "-s", "--format=%ct", commit) + if err != nil { + return Source{}, err + } + epoch, err := strconv.ParseInt(strings.TrimSpace(string(epochOut)), 10, 64) + if err != nil || epoch <= 0 { + return Source{}, errors.New("commit timestamp is invalid") + } + return Source{Commit: commit, Epoch: epoch}, nil +} + +func gitPath(ctx context.Context, runner process.Runner, dir, argument string) (string, error) { + out, err := runner.Run(ctx, dir, nil, "git", "rev-parse", argument) + if err != nil { + return "", err + } + path := strings.TrimSpace(string(out)) + if path == "" { + return "", errors.New("Git returned an empty path") + } + if !filepath.IsAbs(path) { + path = filepath.Join(dir, path) + } + return filepath.Clean(path), nil +} diff --git a/internal/provenance/git_test.go b/internal/provenance/git_test.go new file mode 100644 index 0000000..6f652c9 --- /dev/null +++ b/internal/provenance/git_test.go @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package provenance + +import ( + "context" + "errors" + "strings" + "testing" +) + +type recordingRunner struct { + responses map[string][]byte + calls []string +} + +func (runner *recordingRunner) Run(_ context.Context, _ string, _ map[string]string, name string, args ...string) ([]byte, error) { + key := name + " " + strings.Join(args, " ") + runner.calls = append(runner.calls, key) + response, ok := runner.responses[key] + if !ok { + return nil, errors.New("unexpected command: " + key) + } + return response, nil +} + +func TestInspectAcceptsStandaloneExactPushedCheckout(t *testing.T) { + commit := strings.Repeat("a", 40) + runner := &recordingRunner{responses: map[string][]byte{ + "git rev-parse --git-dir": []byte(".git\n"), + "git rev-parse --git-common-dir": []byte(".git\n"), + "git status --porcelain=v1 --untracked-files=all": nil, + "git rev-parse HEAD": []byte(commit + "\n"), + "git ls-remote --exit-code origin refs/heads/main": []byte(commit + "\trefs/heads/main\n"), + "git show -s --format=%ct " + commit: []byte("1720000000\n"), + }} + result, err := Inspect(context.Background(), runner, "/source", "main") + if err != nil || result.Commit != commit || result.Epoch != 1720000000 { + t.Fatalf("result=%+v err=%v", result, err) + } +} + +func TestInspectExplainsUnsupportedLinkedWorktreeBeforeRemoteOrBuildWork(t *testing.T) { + runner := &recordingRunner{responses: map[string][]byte{ + "git rev-parse --git-dir": []byte("/repo/.git/worktrees/release\n"), + "git rev-parse --git-common-dir": []byte("/repo/.git\n"), + }} + _, err := Inspect(context.Background(), runner, "/source", "main") + if err == nil || !strings.Contains(err.Error(), "linked Git worktrees") || len(runner.calls) != 2 { + t.Fatalf("calls=%#v err=%v", runner.calls, err) + } +} diff --git a/internal/serverpolicy/ownership_linux.go b/internal/serverpolicy/ownership_linux.go new file mode 100644 index 0000000..3fe1303 --- /dev/null +++ b/internal/serverpolicy/ownership_linux.go @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: AGPL-3.0-only +//go:build linux + +package serverpolicy + +import ( + "os" + "syscall" +) + +func rootOwned(info os.FileInfo) bool { + stat, ok := info.Sys().(*syscall.Stat_t) + return ok && stat.Uid == 0 +} diff --git a/internal/serverpolicy/ownership_other.go b/internal/serverpolicy/ownership_other.go new file mode 100644 index 0000000..a4954cf --- /dev/null +++ b/internal/serverpolicy/ownership_other.go @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: AGPL-3.0-only +//go:build !linux + +package serverpolicy + +import "os" + +func rootOwned(os.FileInfo) bool { return false } diff --git a/internal/serverpolicy/policy.go b/internal/serverpolicy/policy.go new file mode 100644 index 0000000..19b4fc9 --- /dev/null +++ b/internal/serverpolicy/policy.go @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package serverpolicy validates the root-owned allowlist used by Tend's +// restricted SSH receiver. It contains service names and paths, never secrets. +package serverpolicy + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "gamertan.com/tend/internal/config" +) + +const SchemaVersion = 1 + +var servicePattern = regexp.MustCompile(`^[a-z][a-z0-9-]{1,62}$`) + +type Policy struct { + SchemaVersion int `json:"schema_version"` + ConfigRoot string `json:"config_root"` + IncomingRoot string `json:"incoming_root"` + SharedLockFile string `json:"shared_lock_file"` + Services map[string]ServicePolicy `json:"services"` +} + +type ServicePolicy struct { + Config string `json:"config"` + MaxArtifactBytes int64 `json:"max_artifact_bytes"` +} + +type CheckedService struct { + Name string + Config config.Config + Policy ServicePolicy +} + +func Parse(reader io.Reader) (Policy, error) { + limited := io.LimitReader(reader, 1<<20+1) + body, err := io.ReadAll(limited) + if err != nil { + return Policy{}, err + } + if len(body) > 1<<20 { + return Policy{}, errors.New("server policy exceeds 1 MiB") + } + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + var policy Policy + if err = decoder.Decode(&policy); err != nil { + return Policy{}, fmt.Errorf("decode server policy: %w", err) + } + var trailing any + if err = decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return Policy{}, errors.New("server policy contains trailing data") + } + if err = policy.Validate(); err != nil { + return Policy{}, err + } + return policy, nil +} + +func Load(path string) (Policy, error) { + if err := secureDirectory(filepath.Dir(path), 0); err != nil { + return Policy{}, fmt.Errorf("server policy directory: %w", err) + } + if err := secureFile(path, 0o600); err != nil { + return Policy{}, fmt.Errorf("server policy: %w", err) + } + file, err := os.Open(path) + if err != nil { + return Policy{}, err + } + defer file.Close() + return Parse(file) +} + +func (policy Policy) Validate() error { + if policy.SchemaVersion != SchemaVersion { + return fmt.Errorf("server policy schema_version must be %d", SchemaVersion) + } + if policy.ConfigRoot != "/etc/tend/services" { + return errors.New("server policy config_root must be /etc/tend/services") + } + if policy.IncomingRoot != "/var/lib/tend/incoming" { + return errors.New("server policy incoming_root must be /var/lib/tend/incoming") + } + if policy.SharedLockFile != config.SharedLockFile { + return fmt.Errorf("server policy shared_lock_file must be %s", config.SharedLockFile) + } + if len(policy.Services) == 0 || len(policy.Services) > 128 { + return errors.New("server policy must allow 1 to 128 services") + } + for name, service := range policy.Services { + if !servicePattern.MatchString(name) { + return fmt.Errorf("invalid service name %q", name) + } + expected := filepath.Join(policy.ConfigRoot, name+".json") + if service.Config != expected { + return fmt.Errorf("service %s config must be %s", name, expected) + } + if service.MaxArtifactBytes < 1<<20 || service.MaxArtifactBytes > 512<<20 { + return fmt.Errorf("service %s artifact limit is invalid", name) + } + } + return nil +} + +func (policy Policy) CheckFiles() ([]CheckedService, error) { + if err := policy.CheckDirectories(); err != nil { + return nil, err + } + names := make([]string, 0, len(policy.Services)) + for name := range policy.Services { + names = append(names, name) + } + sort.Strings(names) + checked := make([]CheckedService, 0, len(names)) + for _, name := range names { + service, err := policy.CheckService(name) + if err != nil { + return nil, err + } + checked = append(checked, service) + } + return checked, nil +} + +func (policy Policy) CheckDirectories() error { + if err := secureDirectory(policy.ConfigRoot, 0); err != nil { + return fmt.Errorf("config root: %w", err) + } + if err := secureDirectory(policy.IncomingRoot, 0o700); err != nil { + return fmt.Errorf("incoming root: %w", err) + } + if err := secureDirectory("/etc/tend/environment", 0o700); err != nil { + return fmt.Errorf("environment root: %w", err) + } + return nil +} + +func (policy Policy) CheckService(name string) (CheckedService, error) { + entry, ok := policy.Services[name] + if !ok { + return CheckedService{}, errors.New("service is not allowed by server policy") + } + if err := secureFile(entry.Config, 0); err != nil { + return CheckedService{}, fmt.Errorf("service %s config: %w", name, err) + } + cfg, err := config.Load(entry.Config) + if err != nil { + return CheckedService{}, fmt.Errorf("service %s config: %w", name, err) + } + if cfg.Service.Name != name { + return CheckedService{}, fmt.Errorf("service %s config identity does not match", name) + } + if cfg.Deployment.LockFile != policy.SharedLockFile { + return CheckedService{}, fmt.Errorf("service %s does not use the host-wide lock", name) + } + if err = secureFile(cfg.Service.EnvironmentFile, 0o600); err != nil { + return CheckedService{}, fmt.Errorf("service %s environment file: %w", name, err) + } + if cfg.Deployment.Singleton != nil { + if err = rejectEnvironmentKey(cfg.Service.EnvironmentFile, cfg.Deployment.Singleton.ListenEnv); err != nil { + return CheckedService{}, fmt.Errorf("service %s environment file: %w", name, err) + } + } + return CheckedService{Name: name, Config: cfg, Policy: entry}, nil +} + +func CheckConfig(path string, cfg config.Config) error { + expected := filepath.Join("/etc/tend/services", cfg.Service.Name+".json") + if path != expected { + return fmt.Errorf("production config must be %s", expected) + } + if err := secureDirectory("/etc/tend/services", 0); err != nil { + return fmt.Errorf("config root: %w", err) + } + if err := secureDirectory("/etc/tend/environment", 0o700); err != nil { + return fmt.Errorf("environment root: %w", err) + } + if err := secureFile(path, 0); err != nil { + return fmt.Errorf("production config: %w", err) + } + if cfg.Deployment.LockFile != config.SharedLockFile { + return fmt.Errorf("deployment lock must be %s", config.SharedLockFile) + } + if err := secureFile(cfg.Service.EnvironmentFile, 0o600); err != nil { + return fmt.Errorf("environment file: %w", err) + } + if cfg.Deployment.Singleton != nil { + if err := rejectEnvironmentKey(cfg.Service.EnvironmentFile, cfg.Deployment.Singleton.ListenEnv); err != nil { + return fmt.Errorf("environment file: %w", err) + } + } + return nil +} + +func rejectEnvironmentKey(path, key string) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + body, err := io.ReadAll(io.LimitReader(file, 1<<20+1)) + if err != nil { + return err + } + if len(body) > 1<<20 { + return errors.New("environment file exceeds 1 MiB") + } + for _, raw := range bytes.Split(body, []byte{'\n'}) { + line := strings.TrimSpace(string(raw)) + if !strings.HasPrefix(line, key) { + continue + } + remainder := strings.TrimSpace(strings.TrimPrefix(line, key)) + if strings.HasPrefix(remainder, "=") { + return errors.New("singleton candidate listen key must not be set in the shared environment file") + } + } + return nil +} + +func secureFile(path string, exactMode os.FileMode) error { + if !filepath.IsAbs(path) || filepath.Clean(path) != path || strings.ContainsRune(path, '\x00') { + return errors.New("path must be clean and absolute") + } + if err := rejectSymlinkAncestors(filepath.Dir(path)); err != nil { + return err + } + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("must be a regular non-symlink file") + } + if !rootOwned(info) { + return errors.New("must be owned by root") + } + if exactMode != 0 && info.Mode().Perm() != exactMode { + return fmt.Errorf("mode must be %04o", exactMode) + } + if exactMode == 0 && info.Mode().Perm()&0o022 != 0 { + return errors.New("must not be group- or world-writable") + } + return nil +} + +func secureDirectory(path string, exactMode os.FileMode) error { + if !filepath.IsAbs(path) || filepath.Clean(path) != path || strings.ContainsRune(path, '\x00') { + return errors.New("path must be clean and absolute") + } + if err := rejectSymlinkAncestors(filepath.Dir(path)); err != nil { + return err + } + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("must be a real directory") + } + if !rootOwned(info) { + return errors.New("must be owned by root") + } + if exactMode != 0 && info.Mode().Perm() != exactMode { + return fmt.Errorf("mode must be %04o", exactMode) + } + if exactMode == 0 && info.Mode().Perm()&0o022 != 0 { + return errors.New("must not be group- or world-writable") + } + return nil +} + +func rejectSymlinkAncestors(path string) error { + current := string(filepath.Separator) + for _, part := range strings.Split(strings.TrimPrefix(filepath.Clean(path), string(filepath.Separator)), string(filepath.Separator)) { + if part == "" { + continue + } + current = filepath.Join(current, part) + info, err := os.Lstat(current) + if err != nil { + return err + } + if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return fmt.Errorf("symlink or non-directory ancestor refused: %s", current) + } + } + return nil +} diff --git a/internal/serverpolicy/policy_test.go b/internal/serverpolicy/policy_test.go new file mode 100644 index 0000000..36c0ad2 --- /dev/null +++ b/internal/serverpolicy/policy_test.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package serverpolicy + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestParseAcceptsStrictServiceMap(t *testing.T) { + body := `{"schema_version":1,"config_root":"/etc/tend/services","incoming_root":"/var/lib/tend/incoming","shared_lock_file":"/run/lock/tend-deploy.lock","services":{"example-site":{"config":"/etc/tend/services/example-site.json","max_artifact_bytes":1048576}}}` + policy, err := Parse(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if policy.Services["example-site"].MaxArtifactBytes != 1<<20 { + t.Fatalf("policy=%+v", policy) + } +} + +func TestRejectEnvironmentKeyProtectsSingletonCandidateOverride(t *testing.T) { + directory := t.TempDir() + path := filepath.Join(directory, "service.env") + for _, test := range []struct { + name string + body string + wantErr bool + }{ + {name: "shared values only", body: "APP_SECRET=private\n"}, + {name: "exact listen key", body: "EXAMPLE_LISTEN=127.0.0.1:8092\n", wantErr: true}, + {name: "spaced listen key", body: " EXAMPLE_LISTEN = 127.0.0.1:8092\n", wantErr: true}, + {name: "commented listen key", body: "# EXAMPLE_LISTEN=127.0.0.1:8092\n"}, + {name: "longer key", body: "EXAMPLE_LISTENER=safe\n"}, + } { + t.Run(test.name, func(t *testing.T) { + if err := os.WriteFile(path, []byte(test.body), 0o600); err != nil { + t.Fatal(err) + } + err := rejectEnvironmentKey(path, "EXAMPLE_LISTEN") + if (err != nil) != test.wantErr { + t.Fatalf("error=%v wantErr=%v", err, test.wantErr) + } + }) + } +} + +func TestParseRejectsUnknownAndForgedPaths(t *testing.T) { + tests := []string{ + `{"schema_version":1,"config_root":"/etc/tend/services","incoming_root":"/var/lib/tend/incoming","shared_lock_file":"/run/lock/tend-deploy.lock","services":{"example-site":{"config":"/tmp/example.json","max_artifact_bytes":1048576}}}`, + `{"schema_version":1,"config_root":"/etc/tend/services","incoming_root":"/var/lib/tend/incoming","shared_lock_file":"/run/lock/tend-deploy.lock","services":{},"surprise":true}`, + } + for _, body := range tests { + if _, err := Parse(strings.NewReader(body)); err == nil { + t.Fatalf("accepted %s", body) + } + } +} diff --git a/internal/state/lease.go b/internal/state/lease.go new file mode 100644 index 0000000..260dbfb --- /dev/null +++ b/internal/state/lease.go @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package state + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/netip" + "os" + "path/filepath" + "regexp" + "time" +) + +const CandidateLeaseSchemaVersion = 1 + +var ( + leaseOperationPattern = regexp.MustCompile(`^[0-9a-f]{32}$`) + leaseServicePattern = regexp.MustCompile(`^[a-z][a-z0-9-]{1,62}$`) +) + +type CandidateLease struct { + SchemaVersion int `json:"schema_version"` + Service string `json:"service"` + OperationID string `json:"operation_id"` + Release string `json:"release"` + Unit string `json:"unit"` + Address string `json:"address"` + StartedAt string `json:"started_at"` +} + +func CandidateLeasePath(statePath string) string { return statePath + ".candidate-lease.json" } + +func (l CandidateLease) Validate(root, service string) error { + if l.SchemaVersion != CandidateLeaseSchemaVersion { + return errors.New("candidate lease schema version is unsupported") + } + if !leaseServicePattern.MatchString(service) || l.Service != service { + return errors.New("candidate lease service does not match configuration") + } + if !leaseOperationPattern.MatchString(l.OperationID) { + return errors.New("candidate lease operation ID is invalid") + } + if err := releaseBelow(root, l.Release); err != nil { + return fmt.Errorf("candidate lease release: %w", err) + } + expectedUnit := service + "-tend-candidate-" + l.OperationID[:12] + ".service" + if l.Unit != expectedUnit { + return errors.New("candidate lease unit does not match its operation") + } + address, err := netip.ParseAddrPort(l.Address) + if err != nil || !address.Addr().IsLoopback() || address.Port() == 0 { + return errors.New("candidate lease address is invalid") + } + if _, err := time.Parse(time.RFC3339, l.StartedAt); err != nil { + return errors.New("candidate lease timestamp is invalid") + } + return nil +} + +func LoadCandidateLease(path, root, service string) (CandidateLease, error) { + info, err := os.Lstat(path) + if err != nil { + return CandidateLease{}, err + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 || info.Size() > 64<<10 { + return CandidateLease{}, errors.New("candidate lease must be a bounded regular file") + } + body, err := os.ReadFile(path) + if err != nil { + return CandidateLease{}, err + } + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + var lease CandidateLease + if err := decoder.Decode(&lease); err != nil { + return CandidateLease{}, fmt.Errorf("decode candidate lease: %w", err) + } + var extra any + if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { + return CandidateLease{}, errors.New("candidate lease contains trailing data") + } + if err := lease.Validate(root, service); err != nil { + return CandidateLease{}, err + } + return lease, nil +} + +func StoreCandidateLease(path, root string, lease CandidateLease) error { + if err := lease.Validate(root, lease.Service); err != nil { + return err + } + if info, err := os.Lstat(path); err == nil && (!info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0) { + return errors.New("candidate lease path must be a regular file, not a symlink") + } else if err != nil && !os.IsNotExist(err) { + return err + } + body, err := json.MarshalIndent(lease, "", " ") + if err != nil { + return err + } + body = append(body, '\n') + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + temporary, err := os.CreateTemp(dir, ".tend-candidate-lease-") + if err != nil { + return err + } + name := temporary.Name() + complete := false + defer func() { + _ = temporary.Close() + if !complete { + _ = os.Remove(name) + } + }() + if err := temporary.Chmod(0o644); err != nil { + return err + } + if _, err := temporary.Write(body); err != nil { + return err + } + if err := temporary.Sync(); err != nil { + return err + } + if err := temporary.Close(); err != nil { + return err + } + if err := os.Rename(name, path); err != nil { + return err + } + complete = true + return syncDir(dir) +} + +func RemoveCandidateLease(path string) error { + info, err := os.Lstat(path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("candidate lease path must be a regular file, not a symlink") + } + if err := os.Remove(path); err != nil { + return err + } + return syncDir(filepath.Dir(path)) +} diff --git a/internal/state/lease_test.go b/internal/state/lease_test.go new file mode 100644 index 0000000..1e0641d --- /dev/null +++ b/internal/state/lease_test.go @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package state + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestCandidateLeaseRoundTripAndRemoval(t *testing.T) { + root := filepath.Join(t.TempDir(), "service") + release := filepath.Join(root, "releases", "sha256-"+strings.Repeat("a", 64)) + if err := os.MkdirAll(release, 0o755); err != nil { + t.Fatal(err) + } + path := CandidateLeasePath(filepath.Join(root, "state.json")) + operation := strings.Repeat("d", 32) + lease := CandidateLease{SchemaVersion: CandidateLeaseSchemaVersion, Service: "example-site", OperationID: operation, Release: release, Unit: "example-site-tend-candidate-" + operation[:12] + ".service", Address: "127.0.0.1:18092", StartedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)} + if err := StoreCandidateLease(path, root, lease); err != nil { + t.Fatal(err) + } + loaded, err := LoadCandidateLease(path, root, "example-site") + if err != nil || loaded != lease { + t.Fatalf("loaded=%+v err=%v", loaded, err) + } + if err := RemoveCandidateLease(path); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(path); !os.IsNotExist(err) { + t.Fatalf("lease remains: %v", err) + } +} + +func TestCandidateLeaseRejectsCrossServiceAndSymlink(t *testing.T) { + root := filepath.Join(t.TempDir(), "service") + release := filepath.Join(root, "releases", "sha256-"+strings.Repeat("a", 64)) + if err := os.MkdirAll(release, 0o755); err != nil { + t.Fatal(err) + } + operation := strings.Repeat("d", 32) + lease := CandidateLease{SchemaVersion: CandidateLeaseSchemaVersion, Service: "example-site", OperationID: operation, Release: release, Unit: "example-site-tend-candidate-" + operation[:12] + ".service", Address: "127.0.0.1:18092", StartedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)} + if err := lease.Validate(root, "other-site"); err == nil { + t.Fatal("expected service mismatch") + } + path := CandidateLeasePath(filepath.Join(root, "state.json")) + if err := os.Symlink(filepath.Join(root, "elsewhere"), path); err != nil { + t.Fatal(err) + } + if err := StoreCandidateLease(path, root, lease); err == nil { + t.Fatal("expected symlink refusal") + } + if err := RemoveCandidateLease(path); err == nil { + t.Fatal("expected symlink removal refusal") + } +} + +func TestCandidateLeasePreservesSchemaOneDeploymentState(t *testing.T) { + root := filepath.Join(t.TempDir(), "service") + release := filepath.Join(root, "releases", "sha256-"+strings.Repeat("a", 64)) + if err := os.MkdirAll(release, 0o755); err != nil { + t.Fatal(err) + } + at := time.Unix(1, 0).UTC().Format(time.RFC3339) + statePath := filepath.Join(root, "state.json") + record := Record{SchemaVersion: SchemaVersion, Strategy: "singleton_candidate", DesiredRelease: release, CandidateRelease: release, ActiveSlot: "singleton", ActiveRelease: release, LastAttemptRelease: release, LastAttemptOutcome: "running", LastAttemptAt: at, UpdatedAt: at} + if err := Store(statePath, root, record); err != nil { + t.Fatal(err) + } + operation := strings.Repeat("d", 32) + lease := CandidateLease{SchemaVersion: CandidateLeaseSchemaVersion, Service: "example-site", OperationID: operation, Release: release, Unit: "example-site-tend-candidate-" + operation[:12] + ".service", Address: "127.0.0.1:18092", StartedAt: at} + if err := StoreCandidateLease(CandidateLeasePath(statePath), root, lease); err != nil { + t.Fatal(err) + } + body, err := os.ReadFile(statePath) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(body, []byte(`"schema_version": 1`)) || bytes.Contains(body, []byte("candidate_operation_id")) || bytes.Contains(body, []byte("candidate_unit")) { + t.Fatalf("deployment state contract changed: %s", body) + } + loaded, err := Load(statePath, root, "singleton_candidate") + if err != nil || loaded.SchemaVersion != 1 || loaded.CandidateRelease != release { + t.Fatalf("loaded=%+v err=%v", loaded, err) + } +} diff --git a/internal/state/state.go b/internal/state/state.go new file mode 100644 index 0000000..7332f85 --- /dev/null +++ b/internal/state/state.go @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package state + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" +) + +const SchemaVersion = 1 + +type Record struct { + SchemaVersion int `json:"schema_version"` + Strategy string `json:"strategy"` + DesiredRelease string `json:"desired_release,omitempty"` + CandidateRelease string `json:"candidate_release,omitempty"` + ActiveSlot string `json:"active_slot"` + ActiveRelease string `json:"active_release"` + PreviousSlot string `json:"previous_slot,omitempty"` + PreviousRelease string `json:"previous_release,omitempty"` + LastAttemptRelease string `json:"last_attempt_release,omitempty"` + LastAttemptOutcome string `json:"last_attempt_outcome,omitempty"` + LastAttemptAt string `json:"last_attempt_at,omitempty"` + UpdatedAt string `json:"updated_at"` +} + +func Load(path, root, strategy string) (Record, error) { + b, err := os.ReadFile(path) + if err != nil { + return Record{}, err + } + dec := json.NewDecoder(bytes.NewReader(b)) + dec.DisallowUnknownFields() + var record Record + if err := dec.Decode(&record); err != nil { + return Record{}, fmt.Errorf("decode state: %w", err) + } + var extra any + if err := dec.Decode(&extra); !errors.Is(err, io.EOF) { + return Record{}, errors.New("state contains trailing data") + } + if err := record.Validate(root, strategy); err != nil { + return Record{}, err + } + return record, nil +} + +func (r Record) Validate(root, strategy string) error { + if r.SchemaVersion != SchemaVersion { + return errors.New("state schema version is unsupported") + } + if r.Strategy != strategy { + return errors.New("state strategy does not match configuration") + } + if strategy == "blue_green" && (r.ActiveSlot != "blue" && r.ActiveSlot != "green") { + return errors.New("active slot is invalid") + } + if strategy == "singleton_candidate" && r.ActiveSlot != "singleton" { + return errors.New("singleton state slot is invalid") + } + if err := releaseBelow(root, r.ActiveRelease); err != nil { + return fmt.Errorf("active release: %w", err) + } + if r.PreviousRelease != "" { + if err := releaseBelow(root, r.PreviousRelease); err != nil { + return fmt.Errorf("previous release: %w", err) + } + } + for label, release := range map[string]string{"desired release": r.DesiredRelease, "candidate release": r.CandidateRelease, "last attempt release": r.LastAttemptRelease} { + if release != "" { + if err := releaseBelow(root, release); err != nil { + return fmt.Errorf("%s: %w", label, err) + } + } + } + if r.LastAttemptOutcome != "" { + switch r.LastAttemptOutcome { + case "running", "succeeded", "failed", "rolled_back": + default: + return errors.New("last attempt outcome is invalid") + } + if _, err := time.Parse(time.RFC3339, r.LastAttemptAt); err != nil { + return errors.New("last attempt timestamp is invalid") + } + } + if r.LastAttemptOutcome == "running" && r.CandidateRelease == "" { + return errors.New("running attempt requires a candidate release") + } + if strategy == "blue_green" && r.PreviousRelease != "" && r.PreviousSlot == r.ActiveSlot { + return errors.New("previous slot must differ from active slot") + } + if _, err := time.Parse(time.RFC3339, r.UpdatedAt); err != nil { + return errors.New("state timestamp is invalid") + } + return nil +} + +func Store(path, root string, record Record) error { + if err := record.Validate(root, record.Strategy); err != nil { + return err + } + if info, err := os.Lstat(path); err == nil && info.Mode()&os.ModeSymlink != 0 { + return errors.New("state file must not be a symlink") + } else if err != nil && !os.IsNotExist(err) { + return err + } + b, err := json.MarshalIndent(record, "", " ") + if err != nil { + return err + } + b = append(b, '\n') + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(dir, ".tend-state-") + if err != nil { + return err + } + name := tmp.Name() + ok := false + defer func() { + _ = tmp.Close() + if !ok { + _ = os.Remove(name) + } + }() + if err := tmp.Chmod(0o644); err != nil { + return err + } + if _, err := tmp.Write(b); err != nil { + return err + } + if err := tmp.Sync(); err != nil { + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Rename(name, path); err != nil { + return err + } + ok = true + return syncDir(dir) +} + +func releaseBelow(root, path string) error { + if !filepath.IsAbs(path) || filepath.Clean(path) != path { + return errors.New("must be a clean absolute path") + } + releases := filepath.Join(root, "releases") + rel, err := filepath.Rel(releases, path) + if err != nil || rel == "." || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || strings.ContainsRune(rel, filepath.Separator) { + return errors.New("must be one direct child of the release directory") + } + return nil +} +func syncDir(path string) error { + dir, err := os.Open(path) + if err != nil { + return err + } + defer dir.Close() + return dir.Sync() +} diff --git a/internal/state/state_test.go b/internal/state/state_test.go new file mode 100644 index 0000000..d2e142a --- /dev/null +++ b/internal/state/state_test.go @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package state + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestStoreLoadRoundTripAndRejectSymlink(t *testing.T) { + root := filepath.Join(t.TempDir(), "service") + release := filepath.Join(root, "releases", "sha256-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + if err := os.MkdirAll(release, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(root, "state.json") + at := time.Unix(1, 0).UTC().Format(time.RFC3339) + record := Record{SchemaVersion: SchemaVersion, Strategy: "singleton_candidate", DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, LastAttemptRelease: release, LastAttemptOutcome: "succeeded", LastAttemptAt: at, UpdatedAt: at} + if err := Store(path, root, record); err != nil { + t.Fatal(err) + } + loaded, err := Load(path, root, "singleton_candidate") + if err != nil { + t.Fatal(err) + } + if loaded.ActiveRelease != release || loaded.DesiredRelease != release || loaded.LastAttemptOutcome != "succeeded" { + t.Fatalf("state=%+v", loaded) + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(root, "elsewhere"), path); err != nil { + t.Fatal(err) + } + if err := Store(path, root, record); err == nil { + t.Fatal("expected symlink refusal") + } +} + +func TestRecordRequiresCandidateForRunningAttempt(t *testing.T) { + root := filepath.Join(t.TempDir(), "service") + release := filepath.Join(root, "releases", "sha256-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + record := Record{SchemaVersion: SchemaVersion, Strategy: "singleton_candidate", DesiredRelease: release, ActiveSlot: "singleton", ActiveRelease: release, LastAttemptRelease: release, LastAttemptOutcome: "running", LastAttemptAt: time.Unix(1, 0).UTC().Format(time.RFC3339), UpdatedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)} + if err := record.Validate(root, "singleton_candidate"); err == nil { + t.Fatal("expected missing candidate rejection") + } +} +func TestRecordRejectsReleaseOutsideRoot(t *testing.T) { + record := Record{SchemaVersion: SchemaVersion, Strategy: "singleton_candidate", ActiveSlot: "singleton", ActiveRelease: "/tmp/other/release", UpdatedAt: time.Unix(1, 0).UTC().Format(time.RFC3339)} + if err := record.Validate("/opt/example", "singleton_candidate"); err == nil { + t.Fatal("expected path refusal") + } +} diff --git a/internal/transport/protocol.go b/internal/transport/protocol.go new file mode 100644 index 0000000..6a4ebf1 --- /dev/null +++ b/internal/transport/protocol.go @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +// Package transport implements Tend's bounded, versioned deployment stream. +package transport + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "strings" +) + +const ( + Protocol = "tend-receive-v1" + MaxArtifactBytes = 512 << 20 + maxHeaderBytes = 64 << 10 + maxArtifactName = 128 +) + +var ( + servicePattern = regexp.MustCompile(`^[a-z][a-z0-9-]{1,62}$`) + artifactPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$`) +) + +type Header struct { + Protocol string `json:"protocol"` + Service string `json:"service"` + ArtifactName string `json:"artifact_name"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + ApprovedSHA256 string `json:"approved_sha256"` + Activate bool `json:"activate"` +} + +func (h Header) Validate(maxBytes int64) error { + if h.Protocol != Protocol { + return errors.New("unsupported receive protocol") + } + if !servicePattern.MatchString(h.Service) { + return errors.New("invalid service name") + } + if len(h.ArtifactName) > maxArtifactName || !artifactPattern.MatchString(h.ArtifactName) { + return errors.New("invalid artifact name") + } + if h.Size <= 0 || h.Size > maxBytes { + return errors.New("artifact size exceeds policy") + } + if h.SHA256 != h.ApprovedSHA256 || len(h.SHA256) != 64 || strings.ToLower(h.SHA256) != h.SHA256 { + return errors.New("artifact digest was not explicitly approved") + } + if _, err := hex.DecodeString(h.SHA256); err != nil { + return errors.New("artifact digest is not hexadecimal") + } + return nil +} + +func Prefix(header Header) ([]byte, error) { + if err := header.Validate(MaxArtifactBytes); err != nil { + return nil, err + } + body, err := json.Marshal(header) + if err != nil { + return nil, err + } + if len(body) > maxHeaderBytes { + return nil, errors.New("receive header exceeds limit") + } + prefix := make([]byte, 4+len(body)) + binary.BigEndian.PutUint32(prefix[:4], uint32(len(body))) + copy(prefix[4:], body) + return prefix, nil +} + +func ReadHeader(reader *bufio.Reader) (Header, error) { + var size [4]byte + if _, err := io.ReadFull(reader, size[:]); err != nil { + return Header{}, fmt.Errorf("read receive header length: %w", err) + } + length := binary.BigEndian.Uint32(size[:]) + if length == 0 || length > maxHeaderBytes { + return Header{}, errors.New("receive header length is invalid") + } + body := make([]byte, length) + if _, err := io.ReadFull(reader, body); err != nil { + return Header{}, fmt.Errorf("read receive header: %w", err) + } + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + var header Header + if err := decoder.Decode(&header); err != nil { + return Header{}, fmt.Errorf("decode receive header: %w", err) + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return Header{}, errors.New("receive header contains trailing data") + } + return header, nil +} + +func CopyArtifact(destination io.Writer, reader *bufio.Reader, header Header, maxBytes int64) error { + if err := header.Validate(maxBytes); err != nil { + return err + } + hash := sha256.New() + written, err := io.CopyN(io.MultiWriter(destination, hash), reader, header.Size) + if err != nil || written != header.Size { + return errors.New("artifact stream ended before declared size") + } + if _, err = reader.ReadByte(); !errors.Is(err, io.EOF) { + return errors.New("artifact stream contains trailing bytes") + } + if hex.EncodeToString(hash.Sum(nil)) != header.SHA256 { + return errors.New("artifact stream digest does not match") + } + return nil +} diff --git a/internal/transport/protocol_test.go b/internal/transport/protocol_test.go new file mode 100644 index 0000000..1f93c58 --- /dev/null +++ b/internal/transport/protocol_test.go @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package transport + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/hex" + "strings" + "testing" +) + +func validFrame(t *testing.T, artifact string) ([]byte, Header) { + t.Helper() + hash := sha256.Sum256([]byte(artifact)) + digest := hex.EncodeToString(hash[:]) + header := Header{Protocol: Protocol, Service: "example-site", ArtifactName: "example.tar.gz", Size: int64(len(artifact)), SHA256: digest, ApprovedSHA256: digest, Activate: true} + prefix, err := Prefix(header) + if err != nil { + t.Fatal(err) + } + return append(prefix, artifact...), header +} + +func TestProtocolRoundTrip(t *testing.T) { + frame, expected := validFrame(t, "artifact") + reader := bufio.NewReader(bytes.NewReader(frame)) + header, err := ReadHeader(reader) + if err != nil || header != expected { + t.Fatalf("header=%+v err=%v", header, err) + } + var artifact bytes.Buffer + if err = CopyArtifact(&artifact, reader, header, 1<<20); err != nil { + t.Fatal(err) + } + if artifact.String() != "artifact" { + t.Fatalf("artifact=%q", artifact.String()) + } +} + +func TestProtocolRejectsTrailingAndForgedInputs(t *testing.T) { + frame, header := validFrame(t, "artifact") + reader := bufio.NewReader(bytes.NewReader(append(frame, 'x'))) + read, _ := ReadHeader(reader) + if err := CopyArtifact(&bytes.Buffer{}, reader, read, 1<<20); err == nil { + t.Fatal("accepted trailing bytes") + } + header.Service = "../../root" + if _, err := Prefix(header); err == nil { + t.Fatal("accepted forged service") + } + header.Service = "example-site" + header.ApprovedSHA256 = strings.Repeat("0", 64) + if _, err := Prefix(header); err == nil { + t.Fatal("accepted unapproved digest") + } +} + +func FuzzProtocolFraming(f *testing.F) { + hash := sha256.Sum256([]byte("artifact")) + digest := hex.EncodeToString(hash[:]) + prefix, err := Prefix(Header{Protocol: Protocol, Service: "example-site", ArtifactName: "example.tar.gz", Size: 8, SHA256: digest, ApprovedSHA256: digest}) + if err != nil { + f.Fatal(err) + } + frame := append(prefix, []byte("artifact")...) + f.Add(frame) + f.Add([]byte{0, 0, 0, 0}) + f.Fuzz(func(t *testing.T, input []byte) { + if len(input) > 2<<20 { + t.Skip() + } + reader := bufio.NewReader(bytes.NewReader(input)) + header, err := ReadHeader(reader) + if err != nil { + return + } + _ = CopyArtifact(&bytes.Buffer{}, reader, header, 1<<20) + }) +} diff --git a/internal/transport/push.go b/internal/transport/push.go new file mode 100644 index 0000000..a02fa7b --- /dev/null +++ b/internal/transport/push.go @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package transport + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" +) + +var targetPattern = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]*@[A-Za-z0-9][A-Za-z0-9.-]*$`) + +type PushOptions struct { + Target string + Port int + KnownHosts string + Identity string + Service string + Artifact string + SHA256 string + ApprovedSHA256 string + Activate bool +} + +type SSHRunner interface { + Run(context.Context, string, []string, io.Reader) ([]byte, error) +} + +type ExecSSHRunner struct{} + +func (ExecSSHRunner) Run(ctx context.Context, name string, args []string, stdin io.Reader) ([]byte, error) { + command := exec.CommandContext(ctx, name, args...) + command.Stdin = stdin + var output, diagnostic boundedBuffer + command.Stdout = &output + command.Stderr = &diagnostic + err := command.Run() + if err != nil { + return output.Bytes(), fmt.Errorf("ssh failed: %w: %s", err, strings.TrimSpace(diagnostic.String())) + } + return output.Bytes(), nil +} + +type boundedBuffer struct{ bytes.Buffer } + +func (b *boundedBuffer) Write(value []byte) (int, error) { + written := len(value) + remaining := (1 << 20) - b.Len() + if remaining > 0 { + if len(value) > remaining { + value = value[:remaining] + } + _, _ = b.Buffer.Write(value) + } + return written, nil +} + +func Push(ctx context.Context, runner SSHRunner, options PushOptions) (json.RawMessage, error) { + if !targetPattern.MatchString(options.Target) || strings.HasPrefix(options.Target, "-") { + return nil, errors.New("target must be user@host without shell syntax") + } + if options.Port < 1 || options.Port > 65535 { + return nil, errors.New("SSH port is invalid") + } + if err := safeClientFile(options.KnownHosts, false); err != nil { + return nil, fmt.Errorf("known-hosts file: %w", err) + } + if options.Identity != "" { + if err := safeClientFile(options.Identity, true); err != nil { + return nil, fmt.Errorf("identity file: %w", err) + } + } + if !filepath.IsAbs(options.Artifact) || filepath.Clean(options.Artifact) != options.Artifact || strings.ContainsAny(options.Artifact, "\x00\r\n\t") { + return nil, errors.New("artifact path must be clean and absolute") + } + artifactInfo, err := os.Lstat(options.Artifact) + if err != nil || !artifactInfo.Mode().IsRegular() || artifactInfo.Mode()&os.ModeSymlink != 0 || artifactInfo.Size() <= 0 || artifactInfo.Size() > MaxArtifactBytes { + return nil, errors.New("artifact must be a bounded regular non-symlink file") + } + artifact, err := os.Open(options.Artifact) + if err != nil { + return nil, err + } + defer artifact.Close() + info, err := artifact.Stat() + if err != nil || !info.Mode().IsRegular() || !os.SameFile(artifactInfo, info) { + return nil, errors.New("artifact must be a bounded regular file") + } + hash := sha256.New() + if _, err = io.Copy(hash, artifact); err != nil { + return nil, err + } + actual := hex.EncodeToString(hash.Sum(nil)) + if actual != options.SHA256 || options.SHA256 != options.ApprovedSHA256 { + return nil, errors.New("artifact digest was not explicitly approved") + } + if _, err = artifact.Seek(0, io.SeekStart); err != nil { + return nil, err + } + header := Header{Protocol: Protocol, Service: options.Service, ArtifactName: filepath.Base(options.Artifact), Size: info.Size(), SHA256: options.SHA256, ApprovedSHA256: options.ApprovedSHA256, Activate: options.Activate} + prefix, err := Prefix(header) + if err != nil { + return nil, err + } + args := []string{"-F", os.DevNull, "-T", "-p", strconv.Itoa(options.Port), "-o", "BatchMode=yes", "-o", "ClearAllForwardings=yes", "-o", "ExitOnForwardFailure=yes", "-o", "ForwardAgent=no", "-o", "IdentitiesOnly=yes", "-o", "LogLevel=ERROR", "-o", "PermitLocalCommand=no", "-o", "ProxyCommand=none", "-o", "RequestTTY=no", "-o", "StrictHostKeyChecking=yes", "-o", "UserKnownHostsFile=" + options.KnownHosts} + if options.Identity != "" { + args = append(args, "-i", options.Identity) + } + args = append(args, options.Target, Protocol) + output, err := runner.Run(ctx, "ssh", args, io.MultiReader(bytes.NewReader(prefix), artifact)) + if err != nil { + return nil, err + } + if !json.Valid(output) { + return nil, errors.New("receiver returned invalid JSON") + } + return json.RawMessage(output), nil +} + +func safeClientFile(path string, private bool) error { + if !filepath.IsAbs(path) || filepath.Clean(path) != path || strings.ContainsAny(path, "\x00\r\n\t") { + return errors.New("path must be clean and absolute") + } + parent, err := os.Lstat(filepath.Dir(path)) + if err != nil || !parent.IsDir() || parent.Mode()&os.ModeSymlink != 0 || parent.Mode().Perm()&0o022 != 0 { + return errors.New("parent must be a real directory not writable by group or others") + } + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Mode()&os.ModeSymlink != 0 { + return errors.New("must be a regular non-symlink file") + } + if info.Mode().Perm()&0o022 != 0 { + return errors.New("must not be group- or world-writable") + } + if private && info.Mode().Perm()&0o077 != 0 { + return errors.New("must not be accessible by group or others") + } + return nil +} diff --git a/internal/transport/push_test.go b/internal/transport/push_test.go new file mode 100644 index 0000000..2ec5b4e --- /dev/null +++ b/internal/transport/push_test.go @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package transport + +import ( + "bufio" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + "slices" + "testing" +) + +type captureRunner struct { + name string + args []string + input []byte + calls int +} + +func (runner *captureRunner) Run(_ context.Context, name string, args []string, input io.Reader) ([]byte, error) { + runner.calls++ + runner.name = name + runner.args = append([]string(nil), args...) + runner.input, _ = io.ReadAll(input) + return []byte(`{"validated":true,"mutation":"activated"}`), nil +} + +func TestPushUsesPinnedSSHAndExactFrame(t *testing.T) { + dir := t.TempDir() + knownHosts := filepath.Join(dir, "known_hosts") + identity := filepath.Join(dir, "identity") + artifact := filepath.Join(dir, "release.tar.gz") + if err := os.WriteFile(knownHosts, []byte("host key\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(identity, []byte("private\n"), 0o600); err != nil { + t.Fatal(err) + } + content := []byte("artifact") + if err := os.WriteFile(artifact, content, 0o600); err != nil { + t.Fatal(err) + } + hash := sha256.Sum256(content) + digest := hex.EncodeToString(hash[:]) + runner := &captureRunner{} + result, err := Push(context.Background(), runner, PushOptions{Target: "tend-deploy@example.test", Port: 2222, KnownHosts: knownHosts, Identity: identity, Service: "example-site", Artifact: artifact, SHA256: digest, ApprovedSHA256: digest, Activate: true}) + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(result, []byte(`"activated"`)) || runner.calls != 1 || runner.name != "ssh" { + t.Fatalf("result=%s calls=%d name=%q", result, runner.calls, runner.name) + } + if !slices.Contains(runner.args, "ProxyCommand=none") || !slices.Contains(runner.args, "StrictHostKeyChecking=yes") || runner.args[len(runner.args)-1] != Protocol { + t.Fatalf("args=%#v", runner.args) + } + reader := bufio.NewReader(bytes.NewReader(runner.input)) + header, err := ReadHeader(reader) + if err != nil { + t.Fatal(err) + } + var copied bytes.Buffer + if err = CopyArtifact(&copied, reader, header, 1<<20); err != nil { + t.Fatal(err) + } + if copied.String() != string(content) || header.Service != "example-site" || !header.Activate { + t.Fatalf("header=%+v body=%q", header, copied.String()) + } +} + +func TestPushRejectsShellTargetBeforeExecution(t *testing.T) { + runner := &captureRunner{} + _, err := Push(context.Background(), runner, PushOptions{Target: "root@example.test;touch", Port: 22}) + if err == nil || runner.calls != 0 { + t.Fatalf("err=%v calls=%d", err, runner.calls) + } +} diff --git a/internal/transport/receive.go b/internal/transport/receive.go new file mode 100644 index 0000000..f1d21b5 --- /dev/null +++ b/internal/transport/receive.go @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package transport + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + + "gamertan.com/tend/internal/deploy" + "gamertan.com/tend/internal/serverpolicy" +) + +func Receive(ctx context.Context, input io.Reader, policy serverpolicy.Policy, manager deploy.Manager) (deploy.Report, error) { + if err := policy.CheckDirectories(); err != nil { + return deploy.Report{}, err + } + reader := bufio.NewReaderSize(input, maxHeaderBytes+4) + header, err := ReadHeader(reader) + if err != nil { + return deploy.Report{}, err + } + service, err := policy.CheckService(header.Service) + if err != nil { + return deploy.Report{}, err + } + if err = header.Validate(service.Policy.MaxArtifactBytes); err != nil { + return deploy.Report{}, err + } + file, err := os.CreateTemp(policy.IncomingRoot, ".tend-receive-"+header.Service+"-") + if err != nil { + return deploy.Report{}, err + } + path := file.Name() + defer os.Remove(path) + if err = file.Chmod(0o600); err == nil { + err = CopyArtifact(file, reader, header, service.Policy.MaxArtifactBytes) + } + if closeErr := file.Close(); err == nil { + err = closeErr + } + if err != nil { + return deploy.Report{}, fmt.Errorf("receive artifact: %w", err) + } + return manager.Deploy(ctx, service.Config, deploy.Request{Artifact: path, SHA256: header.SHA256, ApprovedSHA256: header.ApprovedSHA256, Activate: header.Activate}) +} diff --git a/internal/version/version.go b/internal/version/version.go new file mode 100644 index 0000000..a0c90a8 --- /dev/null +++ b/internal/version/version.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package version + +import ( + "regexp" + "runtime/debug" + "strings" +) + +const developmentVersion = "v0.1.0-dev" + +var ( + Version = developmentVersion + Commit = "unknown" + Date = "unknown" +) + +var ( + taggedVersionPattern = regexp.MustCompile(`^v(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?$`) + pseudoVersionSuffixPattern = regexp.MustCompile(`(?:^|[.-])(?:0\.)?[0-9]{14}-[0-9a-f]{12,}$`) +) + +func init() { + information, ok := debug.ReadBuildInfo() + if !ok { + return + } + Version = selectVersion(Version, information.Main.Version) + for _, setting := range information.Settings { + switch setting.Key { + case "vcs.revision": + if Commit == "unknown" && setting.Value != "" { + Commit = setting.Value + } + case "vcs.time": + if Date == "unknown" && setting.Value != "" { + Date = setting.Value + } + } + } +} + +func selectVersion(linkerValue, moduleVersion string) string { + if linkerValue != developmentVersion { + return linkerValue + } + moduleVersion = strings.TrimSpace(moduleVersion) + if !isTaggedVersion(moduleVersion) { + return linkerValue + } + return moduleVersion +} + +func isTaggedVersion(value string) bool { + matches := taggedVersionPattern.FindStringSubmatch(value) + if matches == nil { + return false + } + prerelease := matches[1] + if pseudoVersionSuffixPattern.MatchString(prerelease) { + return false + } + for _, identifier := range strings.Split(prerelease, ".") { + if len(identifier) > 1 && identifier[0] == '0' && allDecimal(identifier) { + return false + } + } + return true +} + +func allDecimal(value string) bool { + if value == "" { + return false + } + for _, character := range value { + if character < '0' || character > '9' { + return false + } + } + return true +} diff --git a/internal/version/version_test.go b/internal/version/version_test.go new file mode 100644 index 0000000..36a3bad --- /dev/null +++ b/internal/version/version_test.go @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package version + +import "testing" + +func TestSelectVersion(t *testing.T) { + t.Parallel() + tests := []struct { + name, linker, module, want string + }{ + {"local build", developmentVersion, "(devel)", developmentVersion}, + {"missing build info", developmentVersion, "", developmentVersion}, + {"preview one install", developmentVersion, "v0.1.0-preview.1", "v0.1.0-preview.1"}, + {"preview two install", developmentVersion, "v0.1.0-preview.2", "v0.1.0-preview.2"}, + {"release install", developmentVersion, "v1.0.0", "v1.0.0"}, + {"linker override", "v0.1.0-preview.2", "(devel)", "v0.1.0-preview.2"}, + {"pseudo version", developmentVersion, "v0.0.0-20260814120000-0123456789ab", developmentVersion}, + {"pseudo after release", developmentVersion, "v0.1.1-0.20260814120000-0123456789ab", developmentVersion}, + {"build metadata", developmentVersion, "v0.1.0+dirty", developmentVersion}, + {"leading zero release", developmentVersion, "v00.1.0", developmentVersion}, + {"leading zero prerelease", developmentVersion, "v0.1.0-preview.02", developmentVersion}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + if got := selectVersion(test.linker, test.module); got != test.want { + t.Fatalf("selectVersion(%q, %q)=%q want %q", test.linker, test.module, got, test.want) + } + }) + } +} diff --git a/release/tend.json b/release/tend.json new file mode 100644 index 0000000..e3611be --- /dev/null +++ b/release/tend.json @@ -0,0 +1,36 @@ +{ + "schema_version": 2, + "service": { "name": "tend", "allowed_host": "localhost", "environment_file": "/etc/tend/environment/tend-release-test.env" }, + "build": { + "package": "./cmd/tend", + "binary": "tend", + "branch": "main", + "version_symbol": "gamertan.com/tend/internal/version.Version", + "commit_symbol": "gamertan.com/tend/internal/version.Commit", + "date_symbol": "gamertan.com/tend/internal/version.Date" + }, + "deployment": { + "strategy": "singleton_candidate", + "root": "/opt/tend-release-test", + "lock_file": "/run/lock/tend-deploy.lock", + "state_file": "/opt/tend-release-test/tend-state.json", + "event_log": "/opt/tend-release-test/deployment-events.jsonl", + "health_path": "/healthz", + "readiness_path": "/readyz", + "candidate_timeout_seconds": 5, + "activation_window_seconds": 5, + "smoke": [{ "path": "/", "contains": "Tend" }], + "public_smoke": [{ "url": "https://example.test/", "contains": "Tend" }], + "singleton": { + "unit": "tend-release-test.service", + "address": "127.0.0.1:19090", + "candidate_address": "127.0.0.1:19091", + "listen_env": "TEND_RELEASE_TEST_LISTEN", + "current_link": "/opt/tend-release-test/current", + "previous_link": "/opt/tend-release-test/previous", + "caddy_config": "/etc/caddy/Caddyfile", + "caddy_handler": "/etc/caddy/tend-release-test-handler.caddy", + "caddy_handler_template": "/etc/tend/caddy/tend-release-test.template" + } + } +} diff --git a/scripts/check-licenses.sh b/scripts/check-licenses.sh new file mode 100755 index 0000000..e314d1d --- /dev/null +++ b/scripts/check-licenses.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +set -euo pipefail +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +test -s "$root/LICENSE" +test -s "$root/COPYRIGHT" +test -s "$root/examples/LICENSE" +while IFS= read -r file; do + grep -Fq 'SPDX-License-Identifier: AGPL-3.0-only' "$file" || { echo "missing SPDX identifier: $file" >&2; exit 1; } +done < <(find "$root/cmd" "$root/internal" -type f -name '*.go' -print | LC_ALL=C sort) +echo "license boundaries verified" diff --git a/scripts/check-public-tree.sh b/scripts/check-public-tree.sh new file mode 100755 index 0000000..51ca25f --- /dev/null +++ b/scripts/check-public-tree.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +set -euo pipefail +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +allow=$root/scripts/public-snapshot.allow +LC_ALL=C sort -c "$allow" +[[ $(LC_ALL=C sort "$allow" | uniq -d | wc -l) -eq 0 ]] +mapfile -t files <"$allow" +[[ ${#files[@]} -gt 0 ]] +for file in "${files[@]}"; do + [[ -n $file && $file != /* && $file != *..* && $file != .gitea/* && $file != .github/* ]] + git -C "$root" cat-file -e "HEAD:$file" +done +work=$(mktemp -d) +trap 'rm -rf -- "$work"' EXIT +mkdir -m 0700 "$work/tree" +git -C "$root" archive HEAD -- "${files[@]}" | tar -xf - -C "$work/tree" +test ! -e "$work/tree/.git" +test ! -e "$work/tree/.gitea" +test ! -e "$work/tree/.github" +private_pattern='/home/[[:alnum:]_.-]+/|BEGIN (RSA|OPENSSH|EC) PRIVATE KEY|gitea[-_]api[[:alnum:]_.-]*token' +if (cd "$work/tree" && rg -n --hidden --glob '!scripts/export-public.sh' --glob '!scripts/check-public-tree.sh' "$private_pattern" .); then + echo "private material found in public tree" >&2 + exit 1 +fi +(cd "$work/tree" && ./scripts/check-licenses.sh) +(cd "$work/tree" && GOWORK=off go test -count=1 ./...) +(cd "$work/tree" && GOWORK=off go vet ./...) +(cd "$work/tree" && GOWORK=off CGO_ENABLED=0 go build -buildvcs=false -mod=readonly -trimpath -o "$work/tend" ./cmd/tend) +echo "public tree compiles independently" diff --git a/scripts/export-public.sh b/scripts/export-public.sh new file mode 100755 index 0000000..150a4b1 --- /dev/null +++ b/scripts/export-public.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +set -euo pipefail +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +usage(){ echo "Usage: export-public.sh --destination ABSOLUTE-PATH" >&2; } +destination= +while [[ $# -gt 0 ]]; do + case $1 in + --destination) destination=$2; shift 2 ;; + *) usage; exit 2 ;; + esac +done +[[ $destination == /* && $destination != / && ! -e $destination ]] || { usage; exit 2; } +destination=$(realpath -m "$destination");root=$(realpath -e "$root") +case $destination/ in "$root"/*) echo "destination must be outside the private worktree" >&2;exit 1;;esac +git_dir_raw=$(git -C "$root" rev-parse --git-dir) +common_dir_raw=$(git -C "$root" rev-parse --git-common-dir) +[[ $git_dir_raw == /* ]] || git_dir_raw=$root/$git_dir_raw +[[ $common_dir_raw == /* ]] || common_dir_raw=$root/$common_dir_raw +git_dir=$(realpath -e "$git_dir_raw") +common_dir=$(realpath -e "$common_dir_raw") +case $destination/ in "$git_dir"/*|"$common_dir"/*) echo "destination must be outside Git metadata" >&2;exit 1;;esac +[[ -z $(git -C "$root" status --porcelain=v1 --untracked-files=all) ]] || { echo "private worktree is not clean" >&2;exit 1; } +commit=$(git -C "$root" rev-parse HEAD);tree=$(git -C "$root" rev-parse HEAD^{tree}) +[[ $(git -C "$root" ls-remote --exit-code origin refs/heads/main | awk 'NR==1{print $1}') == "$commit" ]] || { echo "private HEAD is not exact pushed origin/main" >&2;exit 1; } +allow=$root/scripts/public-snapshot.allow +LC_ALL=C sort -c "$allow" +[[ $(LC_ALL=C sort "$allow" | uniq -d | wc -l) -eq 0 ]] +mapfile -t files <"$allow" +[[ ${#files[@]} -gt 0 ]] +for file in "${files[@]}"; do + [[ -n $file && $file != /* && $file != *..* && $file != .gitea/* && $file != .github/* ]] + git -C "$root" cat-file -e "$commit:$file" +done +parent=$(dirname "$destination") +mkdir -p "$parent" +stage=$(mktemp -d "$parent/.tend-public.XXXXXX") +trap 'rm -rf -- "$stage"' EXIT +git -C "$root" archive "$commit" -- "${files[@]}" | tar -xf - -C "$stage" +while IFS= read -r file; do + relative=${file#"$stage"/} + cmp "$file" "$root/$relative" +done < <(find "$stage" -type f -print | LC_ALL=C sort) +epoch=$(git -C "$root" show -s --format=%ct "$commit") +printf '{"schema_version":1,"source_commit":"%s","source_tree":"%s","source_date_epoch":%s,"file_count":%d}\n' "$commit" "$tree" "$epoch" "${#files[@]}" >"$stage/PUBLIC-SNAPSHOT.json" +(cd "$stage" && sha256sum PUBLIC-SNAPSHOT.json >PUBLIC-SNAPSHOT.sha256) +private_pattern='/home/[[:alnum:]_.-]+/|BEGIN (RSA|OPENSSH|EC) PRIVATE KEY|gitea[-_]api[[:alnum:]_.-]*token' +if (cd "$stage" && rg -n --hidden --glob '!.git/**' --glob '!PUBLIC-SNAPSHOT.json' --glob '!scripts/export-public.sh' "$private_pattern" .); then + echo "private material found" >&2 + exit 1 +fi +mv "$stage" "$destination" +trap - EXIT +printf 'destination=%s\nsource_commit=%s\nsource_tree=%s\n' "$destination" "$commit" "$tree" diff --git a/scripts/public-snapshot.allow b/scripts/public-snapshot.allow new file mode 100644 index 0000000..67d40d4 --- /dev/null +++ b/scripts/public-snapshot.allow @@ -0,0 +1,81 @@ +.gitattributes +.gitignore +COPYRIGHT +LICENSE +README.md +RELEASE.md +SECURITY.md +cmd/tend/main.go +docs/ARCHITECTURE.md +docs/DOGFOOD_EVIDENCE.md +docs/DOGFOOD_FRICTION.md +docs/PUBLIC_SNAPSHOT.md +docs/SCHEMA_V2_MIGRATION.md +docs/THREAT_MODEL.md +docs/WALKTHROUGH.md +examples/LICENSE +examples/README.md +examples/blue-green/caddy-handler.template +examples/blue-green/example-site@.service +examples/blue-green/tend.json +examples/local/.env.example +examples/server/authorized_keys.example +examples/server/caddy/docs-site.template +examples/server/caddy/example-site.template +examples/server/environment/docs-site.env.example +examples/server/environment/example-site.env.example +examples/server/example-singleton.service +examples/server/receive-policy.json +examples/server/services/docs-site.json +examples/server/services/example-site.json +examples/server/slots/example-site-blue.env +examples/server/slots/example-site-green.env +examples/server/tend-receive.sudoers +examples/singleton/caddy-handler.template +examples/singleton/tend.json +go.mod +internal/config/config.go +internal/config/config_test.go +internal/deploy/deploy.go +internal/deploy/deploy_test.go +internal/deploy/lock_linux.go +internal/deploy/lock_linux_test.go +internal/deploy/lock_other.go +internal/deploy/operator.go +internal/deploy/operator_test.go +internal/deploy/ownership_linux.go +internal/deploy/ownership_other.go +internal/deploy/reconcile.go +internal/deploy/reconcile_test.go +internal/deploy/release.go +internal/deploy/release_mode_linux_test.go +internal/deploy/release_test.go +internal/eventlog/eventlog.go +internal/eventlog/eventlog_test.go +internal/packager/packager.go +internal/packager/packager_test.go +internal/process/run.go +internal/provenance/git.go +internal/provenance/git_test.go +internal/serverpolicy/ownership_linux.go +internal/serverpolicy/ownership_other.go +internal/serverpolicy/policy.go +internal/serverpolicy/policy_test.go +internal/state/lease.go +internal/state/lease_test.go +internal/state/state.go +internal/state/state_test.go +internal/transport/protocol.go +internal/transport/protocol_test.go +internal/transport/push.go +internal/transport/push_test.go +internal/transport/receive.go +internal/version/version.go +internal/version/version_test.go +release/tend.json +scripts/check-licenses.sh +scripts/check-public-tree.sh +scripts/export-public.sh +scripts/public-snapshot.allow +scripts/test-public-snapshot.sh +scripts/verify.sh diff --git a/scripts/test-public-snapshot.sh b/scripts/test-public-snapshot.sh new file mode 100755 index 0000000..fef56ff --- /dev/null +++ b/scripts/test-public-snapshot.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +set -euo pipefail +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +work=$(mktemp -d) +trap 'rm -rf -- "$work"' EXIT +destination=$work/public +"$root/scripts/export-public.sh" --destination "$destination" >/dev/null +test -f "$destination/PUBLIC-SNAPSHOT.json" +test -f "$destination/PUBLIC-SNAPSHOT.sha256" +(cd "$destination" && sha256sum -c PUBLIC-SNAPSHOT.sha256) +test ! -e "$destination/.git" +test ! -e "$destination/.gitea" +test ! -e "$destination/.github" +while IFS= read -r file; do + cmp "$root/$file" "$destination/$file" +done <"$root/scripts/public-snapshot.allow" +echo "public snapshot isolation verified" diff --git a/scripts/verify.sh b/scripts/verify.sh new file mode 100755 index 0000000..89ce5ae --- /dev/null +++ b/scripts/verify.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +set -euo pipefail +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$root" +./scripts/check-licenses.sh +./scripts/check-public-tree.sh +go test -count=1 ./... +go test -race -count=1 ./... +go vet ./... +for script in scripts/*.sh; do bash -n "$script"; done +work=$(mktemp -d); trap 'rm -rf -- "$work"' EXIT +GOWORK=off CGO_ENABLED=0 go build -mod=readonly -trimpath -o "$work/tend-1" ./cmd/tend +GOWORK=off CGO_ENABLED=0 go build -mod=readonly -trimpath -o "$work/tend-2" ./cmd/tend +cmp "$work/tend-1" "$work/tend-2" +"$work/tend-1" check --config "$root/examples/blue-green/tend.json" >/dev/null +"$work/tend-1" check --config "$root/examples/singleton/tend.json" >/dev/null +"$work/tend-1" check --config "$root/examples/server/services/example-site.json" >/dev/null +"$work/tend-1" check --config "$root/examples/server/services/docs-site.json" >/dev/null +"$work/tend-1" check --config "$root/release/tend.json" >/dev/null +grep -Fqx 'Defaults:tend-deploy env_keep += "SSH_ORIGINAL_COMMAND"' "$root/examples/server/tend-receive.sudoers" +grep -Fqx 'tend-deploy ALL=(root) NOPASSWD: /usr/local/bin/tend receive --policy /etc/tend/receive-policy.json' "$root/examples/server/tend-receive.sudoers" +[[ $(GOWORK=off go list -m all | wc -l) -eq 1 ]] +git diff --check +echo "Tend verification passed"